diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000..bf03e01e50
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,147 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+docs/apidocs/nvflare.*
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# pytype cache
+.pytype/
+
+# mypy
+.mypy_cache/
+examples/scd_lvsegs.npz
+temp/
+.idea/
+
+*~
+
+# Remove .pyre temporary config files
+.pyre
+.pyre_configuration
+
+# temporary editor files that should not be in git
+*.orig
+*.bak
+*.swp
+.DS_Store
+
+# temporary testing data MedNIST
+tests/testing_data/MedNIST*
+tests/testing_data/*Hippocampus*
+
+# clang format tool
+.clang-format-bin/
+
+# VSCode
+.vscode/
+
+# TF artifacts
+checkpoint
+ckpt*
+variables.*
+*.pb
+
+# PyTorch artifacts
+*.hdf5
+
+wrksp/*
+
+# poc run data
+*run_*
diff --git a/3rdParty/PyYAML.LICENSE.txt b/3rdParty/PyYAML.LICENSE.txt
new file mode 100644
index 0000000000..2f1b8e15e5
--- /dev/null
+++ b/3rdParty/PyYAML.LICENSE.txt
@@ -0,0 +1,20 @@
+Copyright (c) 2017-2021 Ingy döt Net
+Copyright (c) 2006-2016 Kirill Simonov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/3rdParty/black.LICENSE.txt b/3rdParty/black.LICENSE.txt
new file mode 100644
index 0000000000..7a9b891f71
--- /dev/null
+++ b/3rdParty/black.LICENSE.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Łukasz Langa
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/3rdParty/cryptography.LICENSE.txt b/3rdParty/cryptography.LICENSE.txt
new file mode 100644
index 0000000000..e927e97a2a
--- /dev/null
+++ b/3rdParty/cryptography.LICENSE.txt
@@ -0,0 +1,282 @@
+This software is made available under the terms of *either* of the licenses
+found in LICENSE.APACHE or LICENSE.BSD. Contributions to cryptography are made
+under the terms of *both* these licenses.
+
+The code used in the OpenSSL locking callback and OS random engine is derived
+from CPython, and is licensed under the terms of the PSF License Agreement.
+
+
+========
+1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and
+   the Individual or Organization ("Licensee") accessing and otherwise using Python
+   2.7.12 software in source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+   grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+   analyze, test, perform and/or display publicly, prepare derivative works,
+   distribute, and otherwise use Python 2.7.12 alone or in any derivative
+   version, provided, however, that PSF's License Agreement and PSF's notice of
+   copyright, i.e., "Copyright © 2001-2016 Python Software Foundation; All Rights
+   Reserved" are retained in Python 2.7.12 alone or in any derivative version
+   prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on or
+   incorporates Python 2.7.12 or any part thereof, and wants to make the
+   derivative work available to others as provided herein, then Licensee hereby
+   agrees to include in any such work a brief summary of the changes made to Python
+   2.7.12.
+
+4. PSF is making Python 2.7.12 available to Licensee on an "AS IS" basis.
+   PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.  BY WAY OF
+   EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR
+   WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE
+   USE OF PYTHON 2.7.12 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 2.7.12
+   FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF
+   MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 2.7.12, OR ANY DERIVATIVE
+   THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material breach of
+   its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any relationship
+   of agency, partnership, or joint venture between PSF and Licensee.  This License
+   Agreement does not grant permission to use PSF trademarks or trade name in a
+   trademark sense to endorse or promote products or services of Licensee, or any
+   third party.
+
+8. By copying, installing or otherwise using Python 2.7.12, Licensee agrees
+   to be bound by the terms and conditions of this License Agreement.
+
+========
+Copyright (c) Individual contributors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+    3. Neither the name of PyCA Cryptography nor the names of its contributors
+       may be used to endorse or promote products derived from this software
+       without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+========
+                                 Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/3rdParty/cryptography_APACHE.LICENSE.txt b/3rdParty/cryptography_APACHE.LICENSE.txt
new file mode 100644
index 0000000000..62589edd12
--- /dev/null
+++ b/3rdParty/cryptography_APACHE.LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/3rdParty/cryptography_BSD.LICENSE.txt b/3rdParty/cryptography_BSD.LICENSE.txt
new file mode 100644
index 0000000000..ec1a29d34d
--- /dev/null
+++ b/3rdParty/cryptography_BSD.LICENSE.txt
@@ -0,0 +1,27 @@
+Copyright (c) Individual contributors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+    3. Neither the name of PyCA Cryptography nor the names of its contributors
+       may be used to endorse or promote products derived from this software
+       without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/3rdParty/cryptography_PSF.LICENSE.txt b/3rdParty/cryptography_PSF.LICENSE.txt
new file mode 100644
index 0000000000..4d3a4f57de
--- /dev/null
+++ b/3rdParty/cryptography_PSF.LICENSE.txt
@@ -0,0 +1,41 @@
+1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and
+   the Individual or Organization ("Licensee") accessing and otherwise using Python
+   2.7.12 software in source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+   grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+   analyze, test, perform and/or display publicly, prepare derivative works,
+   distribute, and otherwise use Python 2.7.12 alone or in any derivative
+   version, provided, however, that PSF's License Agreement and PSF's notice of
+   copyright, i.e., "Copyright © 2001-2016 Python Software Foundation; All Rights
+   Reserved" are retained in Python 2.7.12 alone or in any derivative version
+   prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on or
+   incorporates Python 2.7.12 or any part thereof, and wants to make the
+   derivative work available to others as provided herein, then Licensee hereby
+   agrees to include in any such work a brief summary of the changes made to Python
+   2.7.12.
+
+4. PSF is making Python 2.7.12 available to Licensee on an "AS IS" basis.
+   PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.  BY WAY OF
+   EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR
+   WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE
+   USE OF PYTHON 2.7.12 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 2.7.12
+   FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF
+   MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 2.7.12, OR ANY DERIVATIVE
+   THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material breach of
+   its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any relationship
+   of agency, partnership, or joint venture between PSF and Licensee.  This License
+   Agreement does not grant permission to use PSF trademarks or trade name in a
+   trademark sense to endorse or promote products or services of Licensee, or any
+   third party.
+
+8. By copying, installing or otherwise using Python 2.7.12, Licensee agrees
+   to be bound by the terms and conditions of this License Agreement.
diff --git a/3rdParty/flake8-polyfill.LICENSE.txt b/3rdParty/flake8-polyfill.LICENSE.txt
new file mode 100644
index 0000000000..d32256f841
--- /dev/null
+++ b/3rdParty/flake8-polyfill.LICENSE.txt
@@ -0,0 +1,21 @@
+== Flake8 License (MIT) ==
+
+Copyright (C) 2016 Ian Cordasco <graffatcolmingov@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/3rdParty/flake8.LICENSE.txt b/3rdParty/flake8.LICENSE.txt
new file mode 100644
index 0000000000..e5e3d6f940
--- /dev/null
+++ b/3rdParty/flake8.LICENSE.txt
@@ -0,0 +1,22 @@
+== Flake8 License (MIT) ==
+
+Copyright (C) 2011-2013 Tarek Ziade <tarek@ziade.org>
+Copyright (C) 2012-2016 Ian Cordasco <graffatcolmingov@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/3rdParty/google-api-python-client.LICENSE.txt b/3rdParty/google-api-python-client.LICENSE.txt
new file mode 100644
index 0000000000..2987b3b95e
--- /dev/null
+++ b/3rdParty/google-api-python-client.LICENSE.txt
@@ -0,0 +1,22 @@
+ Copyright 2014 Google Inc. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+Dependent Modules
+=================
+
+This code has the following dependencies
+above and beyond the Python standard library:
+
+uritemplates - Apache License 2.0
+httplib2 - MIT License
diff --git a/LICENSE.txt b/3rdParty/grpcio.LICENSE.txt
similarity index 99%
rename from LICENSE.txt
rename to 3rdParty/grpcio.LICENSE.txt
index 261eeb9e9f..d645695673 100644
--- a/LICENSE.txt
+++ b/3rdParty/grpcio.LICENSE.txt
@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
diff --git a/3rdParty/isort.LICENSE.txt b/3rdParty/isort.LICENSE.txt
new file mode 100644
index 0000000000..b5083a50d8
--- /dev/null
+++ b/3rdParty/isort.LICENSE.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Timothy Edmund Crosley
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/3rdParty/keras.LICENSE.txt b/3rdParty/keras.LICENSE.txt
new file mode 100644
index 0000000000..49fc3e7a6a
--- /dev/null
+++ b/3rdParty/keras.LICENSE.txt
@@ -0,0 +1,43 @@
+COPYRIGHT
+
+All contributions by François Chollet:
+Copyright (c) 2015 - 2019, François Chollet.
+All rights reserved.
+
+All contributions by Google:
+Copyright (c) 2015 - 2019, Google, Inc.
+All rights reserved.
+
+All contributions by Microsoft:
+Copyright (c) 2017 - 2019, Microsoft, Inc.
+All rights reserved.
+
+All other contributions:
+Copyright (c) 2015 - 2019, the respective contributors.
+All rights reserved.
+
+Each contributor holds copyright over their respective contributions.
+The project versioning (Git) records all such contribution source information.
+
+LICENSE
+
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
diff --git a/3rdParty/recommonmark.LICENSE.txt b/3rdParty/recommonmark.LICENSE.txt
new file mode 100644
index 0000000000..6ac9d526f1
--- /dev/null
+++ b/3rdParty/recommonmark.LICENSE.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Steve Genoud
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/3rdParty/setuptools.LICENSE.txt b/3rdParty/setuptools.LICENSE.txt
new file mode 100644
index 0000000000..6e0693b4b0
--- /dev/null
+++ b/3rdParty/setuptools.LICENSE.txt
@@ -0,0 +1,19 @@
+Copyright (C) 2016 Jason R Coombs <jaraco@jaraco.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/3rdParty/sphinx.LICENSE.txt b/3rdParty/sphinx.LICENSE.txt
new file mode 100644
index 0000000000..f709c9ad7c
--- /dev/null
+++ b/3rdParty/sphinx.LICENSE.txt
@@ -0,0 +1,192 @@
+License for Sphinx
+==================
+
+Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+Licenses for incorporated software
+==================================
+
+The included smartypants module, included as sphinx.util.smartypants,
+is available under the following license:
+
+----------------------------------------------------------------------
+SmartyPants_ license::
+
+    Copyright (c) 2003 John Gruber
+    (https://daringfireball.net/projects/smartypants/)
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions
+    are met:
+
+    *   Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+
+    *   Redistributions in binary form must reproduce the above
+        copyright notice, this list of conditions and the following
+        disclaimer in the documentation and/or other materials
+        provided with the distribution.
+
+    *   Neither the name "SmartyPants" nor the names of its
+        contributors may be used to endorse or promote products
+        derived from this software without specific prior written
+        permission.
+
+    This software is provided by the copyright holders and
+    contributors "as is" and any express or implied warranties,
+    including, but not limited to, the implied warranties of
+    merchantability and fitness for a particular purpose are
+    disclaimed. In no event shall the copyright owner or contributors
+    be liable for any direct, indirect, incidental, special,
+    exemplary, or consequential damages (including, but not limited
+    to, procurement of substitute goods or services; loss of use,
+    data, or profits; or business interruption) however caused and on
+    any theory of liability, whether in contract, strict liability, or
+    tort (including negligence or otherwise) arising in any way out of
+    the use of this software, even if advised of the possibility of
+    such damage.
+
+
+smartypants.py license::
+
+    smartypants.py is a derivative work of SmartyPants.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions
+    are met:
+
+    *   Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+
+    *   Redistributions in binary form must reproduce the above
+        copyright notice, this list of conditions and the following
+        disclaimer in the documentation and/or other materials
+        provided with the distribution.
+
+    This software is provided by the copyright holders and
+    contributors "as is" and any express or implied warranties,
+    including, but not limited to, the implied warranties of
+    merchantability and fitness for a particular purpose are
+    disclaimed. In no event shall the copyright owner or contributors
+    be liable for any direct, indirect, incidental, special,
+    exemplary, or consequential damages (including, but not limited
+    to, procurement of substitute goods or services; loss of use,
+    data, or profits; or business interruption) however caused and on
+    any theory of liability, whether in contract, strict liability, or
+    tort (including negligence or otherwise) arising in any way out of
+    the use of this software, even if advised of the possibility of
+    such damage.
+----------------------------------------------------------------------
+
+The included JQuery JavaScript library is available under the MIT
+license:
+
+----------------------------------------------------------------------
+Copyright (c) 2008 John Resig, https://jquery.com/
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+----------------------------------------------------------------------
+
+The included Underscore JavaScript library is available under the MIT
+license:
+
+----------------------------------------------------------------------
+Copyright (c) 2009 Jeremy Ashkenas, DocumentCloud
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+-------------------------------------------------------------------------------
+
+The included implementation of NumpyDocstring._parse_numpydoc_see_also_section
+was derived from code under the following license:
+
+-------------------------------------------------------------------------------
+
+Copyright (C) 2008 Stefan van der Walt <stefan@mentat.za.net>, Pauli Virtanen <pav@iki.fi>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+-------------------------------------------------------------------------------
diff --git a/3rdParty/sphinxcontrib-exceltable.LICENSE.txt b/3rdParty/sphinxcontrib-exceltable.LICENSE.txt
new file mode 100644
index 0000000000..5c8e7ba2f3
--- /dev/null
+++ b/3rdParty/sphinxcontrib-exceltable.LICENSE.txt
@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2009 Juha Mustonen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
diff --git a/3rdParty/tenseal.LICENSE.txt b/3rdParty/tenseal.LICENSE.txt
new file mode 100644
index 0000000000..f49a4e16e6
--- /dev/null
+++ b/3rdParty/tenseal.LICENSE.txt
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
\ No newline at end of file
diff --git a/README.md b/README.md
index 77b5639942..6fddb7bd6f 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,25 @@
-# NVFlare
+**NV**IDIA **F**ederated **L**earning **A**pplication **R**untime **E**nvironment
 
-This repo currently contains example codes only.  It intends for users to get a tast of NVFlare by running
-three simple exercises, based on three different computation frameworks, namely PyTorch, Numpy and Tensorflow 2.
 
-The project document page is available [here](https://nvidia.github.io/NVFlare/).
+[NVIDIA FLARE](https://nvidia.github.io/NVFlare) enables researchers to collaborate and build AI models without sharing private data. 
+
+NVIDIA FLARE is a standalone python library designed to enable federated learning amongst different parties using their local secure protected data for client-side training, at the same time it includes capabilities to coordinate and exchange progressing of results across all sites to achieve better global model while preserving data privacy. The participating clients can be in any part of the world. 
+
+NVIDIA FLARE builds on a flexible and modular architecture and is abstracted through APIs allowing developers & researchers to customize their implementation of functional learning components in a Federated Learning paradigm. 
+
+Learn more - [NVIDIA FLARE](https://nvidia.github.io/NVFlare).
+
+
+## Installation
+
+To install [the current release](https://pypi.org/project/nvflare), you can simply run:
+
+```bash
+pip install nvflare
+```
+
+## Third party license
+
+See 3rdParty folder for their license files.
+
 
-NVFlare can be installed via Python pip install.  The PyPi page of NVFlare is available [here](https://pypi.org/project/nvflare/).
diff --git a/build_doc.sh b/build_doc.sh
new file mode 100755
index 0000000000..c2365e46a7
--- /dev/null
+++ b/build_doc.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+# output formatting
+separator=""
+blue=""
+green=""
+red=""
+noColor=""
+
+if [[ -t 1 ]] # stdout is a terminal
+then
+    separator=$'--------------------------------------------------------------------------------\n'
+    blue="$(tput bold; tput setaf 4)"
+    green="$(tput bold; tput setaf 2)"
+    red="$(tput bold; tput setaf 1)"
+    noColor="$(tput sgr0)"
+fi
+
+function print_usage() {
+    echo "build_doc.sh [--clean] [--html]"
+    echo ""
+    echo "Build documentation"
+    echo ""
+    echo "Examples:"
+    echo "./build_doc.sh --html        # build HTML docs."
+    echo "./build_doc.sh --clean       # clean up python build related files."
+}
+
+function print_error_msg() {
+    echo "${red}Error: $1.${noColor}"
+    echo ""
+}
+
+function clean_py() {
+    find nvflare -type f -name "*.py[co]" -delete
+    find nvflare -type f -name "*.so" -delete
+    find nvflare -type d -name "__pycache__" -exec rm -r "{}" +
+
+    find . -depth -maxdepth 1 -type d -name ".eggs" -exec rm -r "{}" +
+    find . -depth -maxdepth 1 -type d -name "*.egg-info" -exec rm -r "{}" +
+    find . -depth -maxdepth 1 -type d -name "build" -exec rm -r "{}" +
+    find . -depth -maxdepth 1 -type d -name "dist" -exec rm -r "{}" +
+    find . -depth -maxdepth 1 -type d -name "__pycache__" -exec rm -r "{}" +
+}
+
+function clean_docs() {
+    find docs -type d -name "_build" -exec rm -r "{}" +
+    find docs/apidocs -type f -name "nvflare*" -delete
+}
+
+function build_html_docs() {
+    # pip install -e ./
+    pip install -r requirements-doc.txt
+    sphinx-apidoc -f -o docs/apidocs/ nvflare "*poc" "*private"
+    sphinx-build -b html docs docs/_build
+}
+
+if [ -z "$1" ]
+then
+    print_error_msg "Too few arguments to $0"
+    print_usage
+fi
+
+# parse arguments
+while [[ $# -gt 0 ]]
+do
+    key="$1"
+    case $key in
+        --clean)
+            doClean=true
+        ;;
+        --html)
+            doHTML=true
+        ;;
+        *)
+            print_error_msg "Incorrect commandline provided, invalid key: $key"
+            print_usage
+        ;;
+    esac
+    shift
+done
+
+if [[ $doClean == true ]]
+then
+    echo "${separator}${blue}clean${noColor}"
+    clean_py
+    clean_docs
+    echo "${green}done!${noColor}"
+    exit
+fi
+
+if [[ $doHTML == true ]]
+then
+    echo "${separator}${blue}html${noColor}"
+    clean_docs
+    build_html_docs
+    echo "${green}done!${noColor}"
+    exit
+fi
diff --git a/doc/Makefile b/doc/Makefile
deleted file mode 100644
index a3450cfc83..0000000000
--- a/doc/Makefile
+++ /dev/null
@@ -1,23 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = source
-BUILDDIR      = build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-clean:
-	rm -rf build/
\ No newline at end of file
diff --git a/doc/_static/custom.css b/doc/_static/custom.css
deleted file mode 100644
index 45875841f8..0000000000
--- a/doc/_static/custom.css
+++ /dev/null
@@ -1,2 +0,0 @@
-@import url('https://fonts.googleapis.com/css?family=Lekton:700|Roboto&display=swap');
-body{font-family:'Roboto',sans-serif;}.wy-side-nav-search>div.version{color:#222;}a:visited{color:#0285b0;}.wy-menu-vertical a:visited{color:#d9d9d9;}.wy-menu-vertical p.caption{color:#7cccc7;}
diff --git a/doc/_templates/footer.html b/doc/_templates/footer.html
deleted file mode 100644
index 8a25214263..0000000000
--- a/doc/_templates/footer.html
+++ /dev/null
@@ -1,20 +0,0 @@
-<footer class="footer mt-5 mt-md-0">
-    <div class="container">
-      <p>
-        {%- if show_copyright %}
-          {% trans copyright=copyright|e %}&copy; {{ copyright }}
-          {% endtrans %} |
-          <a href="https://adap.com/en/imprint">Imprint</a> |
-          <a href="https://adap.com/en/privacy">Privacy Policy</a>
-        {%- endif %}
-        {%- if last_updated %}
-          {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %}<br/>
-        {%- endif %}
-        {%- if theme_extra_footer %}
-          <div class="extra_footer">
-            {{ theme_extra_footer }}
-          </div>
-        {%- endif %}
-      </p>
-    </div>
-  </footer>
diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html
deleted file mode 100644
index 3bf10d958a..0000000000
--- a/doc/_templates/layout.html
+++ /dev/null
@@ -1,85 +0,0 @@
-{% extends "pydata_sphinx_theme/layout.html" %}
-
-{% set is_single_page = (theme_single_page == True) or (theme_single_page == "True") %}
-{% if is_single_page %}
-    {%- set sidebar_width_class = "col-md-2" %}
-{% else %}
-    {%- set sidebar_width_class = "col-md-3" %}
-{% endif %}
-
-{% block extrahead %}
-{{ super() }}
-
-{% if pageurl %}
-<!-- Opengraph tags -->
-<meta property="og:url"         content="{{ pageurl }}" />
-<meta property="og:type"        content="article" />
-<meta property="og:title"       content="{% if pagetitle %}{{ pagetitle | e }}{% else %}{{ docstitle | e }}{% endif %}" />
-<meta property="og:description" content="{{ page_description | e }}" />
-{% if logourl %}<meta property="og:image"       content="{{ logourl }}" />{% endif %}
-
-<meta name="twitter:card" content="summary" />
-{% endif %}
-{% endblock %}
-
-<!-- Docs TOC is "d-none d-xl-block col-xl-2" -->
-
-{% block docs_sidebar %}
-<div class="col-12 {{ sidebar_width_class }} bd-sidebar site-navigation show{% if is_single_page %} single-page{% endif %}" id="site-navigation">
-    {% if not is_single_page %}
-        {% include "sidebar.html" %}
-    {%- endif %}
-</div>
-{% endblock %}
-
-{% block docs_main %}
-<main class="col py-md-3 pl-md-4 bd-content overflow-auto" role="main">
-    {% block docs_body %}
-    {% include "topbar.html" %}
-    <div id="main-content" class="row">
-        <div class="col-12 col-md-9 pl-md-3 pr-md-0">
-        {{ super() }}
-        {% if theme_show_prev_next %}
-        <div class='prev-next-bottom'>
-            {{ prev_next(prev, next) }}
-        </div>
-        {% endif %}
-        </div>
-    </div>
-    {% endblock %}
-    {%- include "footer.html" %}
-</main>
-{% endblock %}
-
-{% block docs_toc %}
-{% endblock %}
-
-{% block docs_navbar %}
-{%- endblock %}
-
-{%- block footer %}
-{%- endblock %}
-
-<script async src="https://www.googletagmanager.com/gtag/js?id=UA-173987939-2"></script>
-<script>
-function onFingerprintJSLoad(fpAgent) {
-    window.dataLayer = window.dataLayer || [];
-    function gtag(){dataLayer.push(arguments);}
-    
-    fpAgent.get().then(result => {
-    gtag('js', new Date());
-    gtag('config', 'UA-173987939-2', {
-        send_page_view: true,
-        client_storage: 'none',
-        client_id: result.visitorId,
-    });
-    });
-}
-
-const script = document.createElement('script');
-script.src = 'https://cdn.jsdelivr.net/npm/@fingerprintjs/fingerprintjs@3/dist/fp.min.js';
-script.id = 'fingerprintjs';
-document.body.appendChild(script);
-
-script.onload = () => FingerprintJS.load().then(onFingerprintJSLoad);
-</script>
diff --git a/doc/_templates/sidebar.html b/doc/_templates/sidebar.html
deleted file mode 100644
index ed130b7418..0000000000
--- a/doc/_templates/sidebar.html
+++ /dev/null
@@ -1,15 +0,0 @@
-<div class="navbar-brand-box">
-    <a class="navbar-brand text-wrap" href="/">
-      {% if logo %}
-      <img src="{{ pathto('_static/' + logo, 1) }}" class="logo" alt="logo">
-      {% endif %}
-      {% if docstitle %}
-      <h1 class="site-logo" id="site-title">{{ docstitle }}</h1>
-      {% endif %}
-    </a>
-    </div>
-    
-    
-    {%- for sidebartemplate in sidebars %}
-    {%- include sidebartemplate %}
-    {%- endfor %}
diff --git a/doc/requirements.txt b/doc/requirements.txt
deleted file mode 100644
index 96f9ad9344..0000000000
--- a/doc/requirements.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-torch>=1.5
-parameterized
-commonmark==0.9.1
-recommonmark==0.6.0
-Sphinx==3.5.3
-sphinx-rtd-theme==0.5.2
-sphinxcontrib-applehelp
-sphinxcontrib-devhelp
-sphinxcontrib-htmlhelp
-sphinxcontrib-jsmath
-sphinxcontrib-qthelp
-sphinxcontrib-serializinghtml
-sphinx-autodoc-typehints==1.11.1
diff --git a/doc/source/conf.py b/doc/source/conf.py
deleted file mode 100644
index af382cdebf..0000000000
--- a/doc/source/conf.py
+++ /dev/null
@@ -1,134 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Path setup --------------------------------------------------------------
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-import os
-import sys
-import subprocess
-
-sys.path.insert(0, os.path.abspath(".."))
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
-print(sys.path)
-
-# import monai  # noqa: E402
-
-# # -- Project information -----------------------------------------------------
-# project = "MONAI"
-# copyright = "2020 - 2021 MONAI Consortium"
-# author = "MONAI Contributors"
-
-# # The full version, including alpha/beta/rc tags
-# short_version = monai.__version__.split("+")[0]
-# release = short_version
-# version = short_version
-
-# # List of patterns, relative to source directory, that match files and
-# # directories to ignore when looking for source files.
-# # This pattern also affects html_static_path and html_extra_path.
-# exclude_patterns = [
-#     "transforms",
-#     "networks",
-#     "metrics",
-#     "engines",
-#     "data",
-#     "apps",
-#     "config",
-#     "handlers",
-#     "losses",
-#     "visualize",
-#     "utils",
-#     "inferers",
-#     "optimizers",
-# ]
-
-
-# def generate_apidocs(*args):
-#     """Generate API docs automatically by trawling the available modules"""
-#     module_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "monai"))
-#     output_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "apidocs"))
-#     apidoc_command_path = "sphinx-apidoc"
-#     if hasattr(sys, "real_prefix"):  # called from a virtualenv
-#         apidoc_command_path = os.path.join(sys.prefix, "bin", "sphinx-apidoc")
-#         apidoc_command_path = os.path.abspath(apidoc_command_path)
-#     print(f"output_path {output_path}")
-#     print(f"module_path {module_path}")
-#     subprocess.check_call(
-#         [apidoc_command_path, "-e"]
-#         + ["-o", output_path]
-#         + [module_path]
-#         + [os.path.join(module_path, p) for p in exclude_patterns]
-#     )
-
-
-# def setup(app):
-#     # Hook to allow for automatic generation of API docs
-#     # before doc deployment begins.
-#     app.connect("builder-inited", generate_apidocs)
-
-
-# -- General configuration ---------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-source_suffix = {".rst": "restructuredtext", ".txt": "restructuredtext", ".md": "markdown"}
-
-extensions = [
-    "recommonmark",
-    "sphinx.ext.intersphinx",
-    "sphinx.ext.mathjax",
-    "sphinx.ext.napoleon",
-    "sphinx.ext.autodoc",
-    "sphinx.ext.viewcode",
-    "sphinx.ext.autosectionlabel",
-    "sphinx_autodoc_typehints",
-]
-
-autoclass_content = "both"
-add_module_names = True
-autosectionlabel_prefix_document = True
-napoleon_use_param = True
-set_type_checking_flag = True
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates"]
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = "sphinx_rtd_theme"
-# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-html_theme_options = {
-    "collapse_navigation": True,
-    "display_version": True,
-    "sticky_navigation": True,  # Set to False to disable the sticky nav while scrolling.
-    "logo_only": True,  # if we have a html_logo below, this shows /only/ the logo with no title text
-    "style_nav_header_background": "#FBFBFB",
-}
-html_context = {
-    "display_github": False,
-    "github_user": "NVIDIA",
-    "github_repo": "hello_nvflare",
-    "github_version": "main",
-    "conf_py_path": "/doc/",
-}
-html_scaled_image_link = False
-html_show_sourcelink = True
-# html_favicon = "../images/favicon.ico"
-# html_logo = "../images/MONAI-logo-color.png"
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["../_static"]
-html_css_files = ["custom.css"]
diff --git a/doc/source/hello_numpy.rst b/doc/source/hello_numpy.rst
deleted file mode 100644
index 63046f1e39..0000000000
--- a/doc/source/hello_numpy.rst
+++ /dev/null
@@ -1,293 +0,0 @@
-Quickstart (Numpy)
-===================
-
-Before You Start
-----------------
-
-Before jumping into this QuickStart guide, make sure you have an environment with `NVFlare <https://pypi.org/project/nvflare/>`_ installed.  You can follow
-:doc:`hello_installation` on the general concept of Python virtual environment (the recommended environment) and how to
-install NVFlare.
-
-
-Introduction
--------------
-
-This tutorial is meant to solely demonstrate how the NVFlare system works, without introducing any actual deep learning concepts.
-Through this exercise, you will learn how to use NVFlare with numpy to perform basic computations across two clients.
-Due to the simplified weights, you will be able to clearly see and understand the results of the FL aggregation and the model persistor process.
-
-The design of this exercise consists of one **server** and two **clients**, starting with weights ``[0, 1]``.
-The following steps compose one cycle of weight updates, called a **round**:
-
- #. Clients are responsible for adding (with some slight errors) the two weights together to calculate a Fibonacci number for the model. 
- #. These updates are then sent to the server which will aggregate them (averaging out the errors) to produce a model with new weights. 
- #. Finally, the server sends this updated version of the model back to each client, so the clients can continue to calculate the next Fibonacci number in future rounds.
-
-For this exercise, we will be working with the ``hello-numpy`` application in the examples folder.
-Every custom FL application must contain three folders:
-
- #. **custom**: contains the custom components (``trainer.py``, ``model_persistor.py``)
- #. **config**: contains client and server configurations (``config_fed_client.json``, ``config_fed_server.json``)
- #. **resources**: contains the logger config (``log.config``)
-
-Let's get started. First clone the repo, if you haven't already:
-
-.. code-block:: shell
-
-  $ git clone https://github.com/nvidia/nvflare.git
-
-Remember to activate your NVFlare Python virtual environment from the installation guide. Ensure numpy is installed.
-
-.. code-block:: shell
-
-  (nvflare-env) $ python3 -m pip install numpy
-
-Now that you have all your dependencies installed, let's implement the Federated Learning system.
-
-
-NVFlare Client
---------------
- 
-In a file called ``trainer.py``, import nvflare and numpy.
-Now we will implement the train function to enable the clients to perform
-a simple addition of two numbers to represent one calculation in the Fibonacci Sequence.
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-numpy/custom/trainer.py
-   :language: python
-   :linenos:
-   :caption: trainer.py
-
-
-The server sends either the initial weights of [0, 1] or any stored weights to each of the clients
-through the ``Shareable`` object passed into ``train()``. Each client adds the
-two numbers together, and creates a new ``Shareable`` to include the sum of the two 
-numbers as well as the second number of original shareable.  
-This represents the calculation of one term in the Fibonacci Sequence:
-
-``[a, b] => [b, a + b]``
-wherein ``a`` and ``b`` are two consecutive Fibonacci numbers, with ``b > a``.
-
-In a real federated learning training scenario, each client does its training independently on its own dataset. 
-As such, the weights returned to the server will likely be different for each of the clients. 
-Because this exercise is simply doing an addition operation rather than a training on a dataset, we
-**hard-code** a slight variation in the clients' addition to demonstrate how the server aggregation works. 
-We define that client-1 always **subtracts** one extra from every element, while client-2 always 
-**adds** one extra to every element.
-
-Fortunately, because this is a FL system with two clients, the server can ``aggregrate``
-(in this case average) the clients' results to produce the correct Fibonacci number.
-
-You can learn more about ``Shareable`` and ``FLContext`` in the :doc:`hello_pt` exercise or in the :ref:`documentation <programming_guide:Key Objects>`.
-
-
-NVFlare Server & Application
-------------------------------
-
-Model Persistor
-^^^^^^^^^^^^^^^^^
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-numpy/custom/model_persistor.py
-   :language: python
-   :linenos:
-   :caption: model_persistor.py
-
-The model persistor is used to load and save models on the server. Here, the model is a ``Model`` object containing two calculated Fibonacci numbers.
-
-The expected format of the ``Model`` object is a dictionary with keys of type ``str`` and values of type ``numpy.array``. 
-This can be easily changed as long as all of the FL components agree on the format.
-
-In this exercise, we can simply serialize the model dictionary using pickle and save it to weights.pickle 
-inside the ``server/run_1/mmar_server/models/`` directory (weights file name specified in ``config_fed_client.json``).
-Depending on the frameworks and tools, the methods of saving the model may vary.
-
-.. note::
-
-  Note that in our hello-examples, we are demonstrating Federated Learning using the familiar deep learning model concept.
-
-  However, Federated Learning is not restricted to just deep learning and may not always involve models. Thus, 
-  we define a :ref:`Learnable <programming_guide:Learnable>` object (subclasses ``dict``) as the most general form of data object generated through Federated Learning.
-  The ``Model`` class we are using here is simply a subclass of ``Learnable``.
-
-
-Application Configuration
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Inside the config folder there are two files, ``config_fed_client.json`` and ``config_fed_server.json``.
-For now, the default configurations are sufficient.
-
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-numpy/config/config_fed_server.json
-   :language: json
-   :linenos:
-   :caption: config_fed_server.json
-
-
-Note that the ``persistor`` points to the custom ``NumpyModelPersistor`` with full Python module path.
-
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-numpy/config/config_fed_client.json
-   :language: json
-   :linenos:
-   :caption: config_fed_client.json
-
-
-Here, the ``client_trainer`` points to the Trainer implementation ``SimpleTrainer``.
-
-
-Federated Numpy Fibonacci!
---------------------------
-
-Now you must set up a local environment and generate packages to simulate the server, clients, and admin.
-
-Setting Up the Application Environment
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-This command generates a poc folder with server, one client and one admin:
-
-.. code-block:: shell
-
-    $ poc
-
-Here we duplicate the client folder into two folders to create two clients, site-1 and site-2:
-
-.. code-block:: shell
-
-    $ cp -r poc/client poc/site-1
-    $ cp -r poc/client poc/site-2
-
-Finally, we copy necessary files (the exercise codes) to a working folder:
-
-.. code-block:: shell
-
-  $ mkdir -p poc/admin/transfer
-  $ cp -rf examples/* poc/admin/transfer
-
-With both the client and server ready, you can now run everything and see federated
-learning in action. FL systems usually have a server and multiple clients. We
-therefore have to start the server first:
-
-.. code-block:: shell
-
-    $ ./poc/server/startup/start.sh
-
-Once the server is running you can start the clients in different terminals.
-Open a new terminal and start the first client:
-
-.. code-block:: shell
-
-    $ ./poc/site-1/startup/start.sh site-1 localhost
-
-Open another terminal and start the second client:
-
-.. code-block:: shell
-
-    $ ./poc/site-2/startup/start.sh site-2 localhost
-
-In one last terminal, start the admin:
-
-.. code-block:: shell
-
-  $ ./poc/admin/startup/fl_admin.sh localhost
-
-
-This will launch a command prompt, where you can input commands to control and monitor many aspects of
-the FL process. Log in by entering ``admin`` for both the username and password.
-
-Running the FL
-^^^^^^^^^^^^^^
-
-Enter the commands below in order.  Pay close attention to what happens in each of four terminals.  You
-can see the admin controls server and clients with each command.
-
-.. code-block:: shell
-
-    > upload_app hello-numpy
-
-Uploads an application in the server's registry.  This creates the application entry, populates the configuration and links the name
-``hello-numpy`` with such application configuration.  Later, you can control this application via this name.
-
-.. code-block:: shell
-
-    > set_run_number 1
-
-Creates a workspace with the run_number on the server and all clients.  The purpose of this workspace is to isolate different runs so
-the information in one particular run does not interfere with other runs.
-
-.. code-block:: shell
-
-    > deploy_app hello-numpy all
-
-This will make the hello-numpy application the active one in the run_number workspace.  After the above two commands, the
-server and all the clients know the hello-numpy application will reside in the ``run_1`` workspace.
-
-
-.. code-block:: shell
-
-    > start_app all
-
-This ``start_app`` command instructs the NVFlare server and clients to start training with the hello-numpy application in that ``run_1`` workspace.
-
-From time to time, you can issue ``check_status server`` in the admin client to check the entire training progress.
-
-You should now see how the training does in the very first terminal (the one that started the server):
-
-
-Understanding the Output
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-After starting the server and clients, you should begin to see 
-some outputs in each terminal tracking the progress of the fl run. If everything went as 
-planned, you should see that through 10 rounds, the FL system has calculated the correct 
-numbers in the Fibonacci Sequence: ``0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89``
-
-Here are some snippets of the output that you can follow along with. 
-
-.. code-block:: shell
-    :caption: site-1
-
-    Get global model for round: 6
-    pull_models completed. Status:True rank:0
-    site-1 Shareable from server:  {'sequence': array([ 8., 13.])}
-    site-1 Updated Shareable to server:  {'sequence': array([12., 20.])}
-
-    Send model to server.
-    2021-07-01 16:22:28,431 - FederatedClient - INFO - Starting to push model.
-    2021-07-01 16:22:28,432 - Communicator - INFO - Send example_project at round 6
-
-.. code-block:: shell
-    :caption: site-2
-
-    Get global model for round: 6
-    pull_models completed. Status:True rank:0
-    site-2 Shareable from server:  {'sequence': array([ 8., 13.])}
-    site-2 Updated Shareable to server:  {'sequence': array([14., 22.])}
-
-    Send model to server.
-    2021-07-01 16:22:31,462 - FederatedClient - INFO - Starting to push model.
-    2021-07-01 16:22:31,463 - Communicator - INFO - Send example_project at round 6
-
-.. code-block:: shell
-    :caption: server
-
-    2021-07-01 16:22:38,453 - FederatedServer - INFO - > aggregating: 6
-    2021-07-01 16:22:38,453 - AccumulateWeightedAggregator - INFO - aggregating 2 updates at round 6
-    Saving aggregated server weights:  {'sequence': array([13., 21.])}
-
-As you can see, on round 6 both of the clients receive ``[8, 13]`` from the server. However, instead of sending back ``[13, 8 + 13] = [13, 21]``, due to the built in biases, site-1 sends back ``[12, 20]`` and site-2 sends back ``[14, 22]`` to the server.
-
-Finally, through aggregating the clients' results, the server saves the weights for round 6 with the correct sequence of ``[13, 21]``.
-
-Once the fl run is complete and the server has successfully aggregrated the client's results after all the rounds, run the following commands in the fl_admin to shutdown the system (while inputting ``admin`` when prompted with password):
-
-.. code-block:: shell
-
-    > shutdown client
-    > shutdown server
-    > bye
-
-In order to stop all processes, run ``./stop_fl.sh``. 
-
-If you want to reset the saved server weights to start from ``[0,1]`` again, delete ``weights.pickle`` in the ``server/run_1/mmar_server/models/`` directory.
-
-Congratulations!
-You've successfully built and run your first numpy federated learning system. 
-You now have a decent grasp of the main FL concepts, and are ready to start exploring how NVFlare can be applied to many other tasks.
diff --git a/doc/source/hello_tf2.rst b/doc/source/hello_tf2.rst
deleted file mode 100644
index 8d7b760f81..0000000000
--- a/doc/source/hello_tf2.rst
+++ /dev/null
@@ -1,366 +0,0 @@
-Quickstart (TensorFlow 2)
-===========================
-
-Before You Start
-----------------
-
-We recommend you first finish either the :doc:`hello_pt` or the :doc:`hello_numpy` exercise.
-Those guides go more in depth in explaining the federated learning aspect of `NVFlare <https://pypi.org/project/nvflare/>`_. 
- 
-Here we assume you have already installed NVFlare inside a python virtual environment and have already cloned the repo.
-
-Introduction
--------------
-
-Through this exercise, you will integrate NVFlare with the popular
-deep learning framework `TensorFlow 2 <https://www.tensorflow.org/>`_ and learn how to use NVFlare to train a convolutional network with the MNIST dataset.
-You will also be introduced to some new components and concepts, including the filter, aggregrator, and event handler.
-
-The design of this exercise consists of one **server** and two **clients** all having the same TensorFlow 2 model. 
-The following steps compose one cycle of weight updates, called a **round**:
-
- #. Clients are responsible for generating individual weight-updates for the model using their own MNIST dataset. 
- #. These updates are then sent to the server which will aggregate them to produce a model with new weights. 
- #. Finally, the server sends this updated version of the model back to each client.
-
-For this exercise, we will be working with the ``hello-tf2`` application in the examples folder. 
-Every custom FL application must contain three folders:
-
- #. **custom**: contains the custom components (``net.py``, ``trainer.py``, ``filter.py``, ``model_aggregator.py``, ``model_persistor.py``)
- #. **config**: contains client and server configurations (``config_fed_client.json``, ``config_fed_server.json``)
- #. **resources**: contains the logger config (``log.config``)
-
-Let's get started. Since this task is using TensorFlow, let's go ahead and install the library inside our virtual environment: 
-
-.. code-block:: shell
-
-  (nvflare-env) $ python3 -m pip install tensorflow
-
-
-NVFlare Client
---------------
-
-Network
-^^^^^^^^
-
-With all the required dependencies installed, you are ready to run a Federated Learning
-with two clients and one server. Before you start, let's see what a simplified MNIST network looks like.
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-tf2/custom/net.py
-   :language: python
-   :linenos:
-   :caption: net.py
-
-This Net class is the convolutional neural network to train with MNIST dataset. This is not related to NVFlare, so implement it in a file called ``net.py``.
-
-Dataset & Setup
-^^^^^^^^^^^^^^^^
-
-Now you have to implement the only class related to NVFlare, ``Trainer``, in a file called ``trainer.py``.
-
-Before you can really start a training, you need to setup your dataset.
-In this exercise, you can download it from the Internet via ``tf.keras``'s datasets module, and split it in half to create a separate dataset for each client.
-Additionally, you must setup the optimizer, loss function and transform to process the data.
-
-Since every step will be encapsulated in the ``Trainer`` class, let's put this preparation stage into one method ``setup``:
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-tf2/custom/trainer.py
-   :language: python
-   :lines: 24-53
-   :lineno-start: 24
-   :linenos:
-
-
-How can you ensure this setup method is called before the client receives the model from the server?  The Trainer
-class is also a :ref:`FLComponent <programming_guide:FLComponent>`, which always receives ``Event`` whenever NVFlare enters or leaves a certain stage.
-In this case, there is an ``Event`` called ``EventType.START_RUN`` which perfectly matches these requirements. 
-Because our trainer is a subclass of ``FLComponent``, you can implement the handler to handle the event and call the setup method:
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-tf2/custom/trainer.py
-   :language: python
-   :lines: 20-22
-   :lineno-start: 20
-   :linenos:
-
-.. note::
-
-  This is a new concept you haven't learned in previous two exercises.  The concepts about ``event`` and ``handler`` are very powerful because
-  you are free to add your logic so it can run at different time and process various events.  The entire list of events fired by
-  NVFlare is shown at :ref:`Event types <programming_guide:Event types>`.
-
-
-You have everything you need, now let's implement the last method called ``train``, which is 
-called every time the client receives an updated model from the server.
-
-
-Link NVFlare with Local Train
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Take a look at the following code:
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-tf2/custom/trainer.py
-   :language: python
-   :lines: 55-
-   :lineno-start: 55
-   :linenos:
-
-Every NVFlare client receives the model weights from the server in the :ref:`shareable <programming_guide:Shareable>`.
-This exercise uses a simple ``exclude_var`` filter, so make sure to replace the missing layer with weights from the clients' previous training round:
-
-.. code-block:: python
-
-  for key, value in model_weights.items():
-        if np.all(value == 0):
-            model_weights[key] = prev_weights[key]
-
-Now update the local model with those received weights:
-
-.. code-block:: python
-
-  self.model.set_weights(list(model_weights.values()))
-
-Then peform a simple :code:`self.model.fit` so the client's model is trained with its own dataset:
-
-.. code-block:: python
-
-    self.model.fit(
-        self.train_images,
-        self.train_labels,
-        epochs=self.epochs_per_round,
-        validation_data=(self.test_images, self.test_labels),
-    )
-  
-After finishing the local train, the train method builds a new ``shareable`` with newly-trained weights and returns it
-back to the NVFlare server.
-
-
-NVFlare Server & Application
------------------------------
-
-Filter
-^^^^^^^ 
-
-The :ref:`filter <programming_guide:Filters>` is used for additional data processing in the ``shareable``, either inbound or outbound from the client and/or server. 
-
-For this exercise, we use a basic ``exclude_var`` filter to exclude the variable/layer ``flatten`` outbound from the client to the server.
-The excluded layer is replaced with all zeros of the same shape, which reduces compression size and ensures that the clients' weights
-for this variable are not shared with the server.
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-tf2/custom/filter.py
-   :language: python
-   :linenos:
-   :caption: filter.py
-
-The filtering procedure occurs in the one required method, process, which receives and returns a shareable.
-The parameters for what is excluded and the inbound/outbound option are all set in ``config_fed_client.json`` (shown later below) and passed in through the constructor.
-
-
-Model Aggregator
-^^^^^^^^^^^^^^^^
-
-The :ref:`model aggregator <programming_guide:Aggregator>` is used by the server to aggregrate the clients' models into one model.
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-tf2/custom/model_aggregator.py
-   :language: python
-   :linenos:
-   :caption: model_aggregator.py
-
-In this exercise, we perform a simple average over the two clients' weights. 
-
-The ``accept()`` function is responsible for managing the shareables from the clients. 
-Note that the second return boolean can be set to ``True`` based on some logic to call ``aggregate()`` early, 
-otherwise by default ``aggregate()`` will always be called once the number of submissions is >= ``min_clients``.
-
-The ``aggregrate()`` function then performs the aggregration procedure and returns a new shareable.
-
-FLContext is used throughout these functions to provide various useful FL-related information. 
-You can find more details in the :ref:`documentation <programming_guide:FLContext>`.
-
-Model Persistor
-^^^^^^^^^^^^^^^
-
-The model persistor is used to load and save models on the server.
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-tf2/custom/model_persistor.py
-   :language: python
-   :linenos:
-   :caption: model_persistor.py
-
-In this exercise, we simply serialize the model weights dictionary using pickle and save it to ``server/run_1/mmar_server/models/tf2weights.pickle``
-inside the server directory (weights file name defined in ``config_fed_server.json``).
-Depending on the frameworks and tools, the methods of saving the model may vary.
-
-Application Configuration
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Finally, inside the config folder there are two files, ``config_fed_client.json`` and ``config_fed_server.json``.
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-tf2/config/config_fed_server.json
-   :language: json
-   :linenos:
-   :caption: config_fed_server.json
-
-
-Note that the ``aggregator`` points to the custom ``AccumulateAggregator``, 
-and the ``persistor`` points to the custom ``TF2ModelPersistor`` with full Python module paths. 
-
-
-.. literalinclude:: ../../hello_nvflare/examples/hello-tf2/config/config_fed_client.json
-   :language: json
-   :linenos:
-   :caption: config_fed_client.json
-
-
-Here, the ``client_trainer`` points to the Trainer implementation ``SimpleTrainer``. 
-Also we set the outbound filter path to ``filter.ExcludeVars`` and pass in ``["flatten"]`` as the argument.
-
-Train the Model, Federated!
----------------------------
-
-Now you must set up a local environment and generate packages to simulate the server, clients, and admin.
-
-Setting Up the Application Environment
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-This command generates a poc folder with server, one client and one admin:
-
-.. code-block:: shell
-
-    $ poc
-
-Here we duplicate the client folder into two folders to create two clients, site-1 and site-2:
-
-.. code-block:: shell
-
-    $ cp -r poc/client poc/site-1
-    $ cp -r poc/client poc/site-2
-
-Finally, we copy necessary files (the exercise codes) to a working folder:
-
-.. code-block:: shell
-
-  $ mkdir -p poc/admin/transfer
-  $ cp -rf examples/* poc/admin/transfer
-
-With both the client and server ready, you can now run everything and see federated
-learning in action. FL systems usually have a server and multiple clients. We
-therefore have to start the server first:
-
-.. code-block:: shell
-
-    $ ./poc/server/startup/start.sh
-
-Once the server is running you can start the clients in different terminals.
-Open a new terminal and start the first client:
-
-.. code-block:: shell
-
-    $ ./poc/site-1/startup/start.sh site-1 localhost
-
-Open another terminal and start the second client:
-
-.. code-block:: shell
-
-    $ ./poc/site-2/startup/start.sh site-2 localhost
-
-In one last terminal, start the admin:
-
-.. code-block:: shell
-
-  $ ./poc/admin/startup/fl_admin.sh localhost
-
-
-This will launch a command prompt, where you can input commands to control and monitor many aspects of
-the FL process. Log in by entering ``admin`` for both the username and password.
-
-Running the FL
-^^^^^^^^^^^^^^
-
-Enter the commands below in order.  Pay close attention to what happens in each of four terminals.  You
-can see the admin controls server and clients with each command.
-
-.. code-block:: shell
-
-    > upload_app hello-tf2
-
-Uploads an application in the server's registry.  This creates the application entry, populates the configuration and links the name
-``hello-tf2`` with such application configuration.  Later, you can control this application via this name.
-
-.. code-block:: shell
-
-    > set_run_number 1
-
-Creates a workspace with the run_number on the server and all clients.  The purpose of the workspace is to isolate different runs so
-the information in one particular run does not interfere with other runs.
-
-.. code-block:: shell
-
-    > deploy_app hello-tf2 all
-
-This will make the hello-tf2 application the active one in the run_number workspace.  In this exercise, after the above two commands, the
-server and all the clients know the hello-tf2 application will reside in ``run_1`` workspace.
-
-
-.. code-block:: shell
-
-    > start_app all
-
-This ``start_app`` command instructs the NVFlare server and clients to start training with the hello-tf2 application in that ``run_1`` workspace.
-
-From time to time, you can issue ``check_status server`` in the admin client to check the entire training progress.
-
-You should now see how the training does in the very first terminal (the one that started the server):
-
-
-.. code-block:: shell
-
-    2021-07-02 14:25:41,361 - FederatedServer - INFO - starting secure server at localhost:8002
-    deployed FL server trainer.
-    2021-07-02 14:25:41,366 - FedAdminServer - INFO - Starting Admin Server localhost on Port 8003
-    2021-07-02 14:25:41,366 - root - INFO - Server started
-    2021-07-02 14:25:50,530 - ClientManager - INFO - Client: New client site-1@127.0.0.1 joined. Sent token: fd9e29f-cb4a-4162-b684-b16be3958830.  Total clients: 1
-    2021-07-02 14:25:58,435 - ClientManager - INFO - Client: New client site-2@127.0.0.1 joined. Sent token: 42fa2ef-5330-4c2f-9639-77ceac1f6db1.  Total clients: 2
-    Create initial model message...
-    created initial model_data...
-    2021-07-02 14:27:01,586 - FederatedServer - INFO - Server training has been started.
-    2021-07-02 14:27:13,344 - FederatedServer - INFO - GetModel requested from: fd96e29f-cb4a-4162-b684-b16be395830
-    2021-07-02 14:27:13,347 - FederatedServer - INFO - Return model to : fd96e29f-cb4a-4162-b684-b16be3958830 fo round: 0
-
-On your client terminal:
-
-.. code-block:: shell
-
-    starting the client .....
-    token is: 42f6a2ef-5330-4c2f-9639-77ceac1f6db1 run_number is: 1 uid: site-2 listen_port: 46465
-    Created the listener on port: 46465
-    2021-07-02 14:27:14,772 - FederatedClient - INFO - Starting to fetch global model.
-    2021-07-02 14:27:14,825 - Communicator - INFO - Received example_project model at round 0 (407809 Bytes). GetModel time: 0.04601311683654785 seconds
-    2021-07-02 14:27:15.154289: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:176] None of the MLIR Optimization Passes are enabled (registered 2)
-    2021-07-02 14:27:15.174696: I tensorflow/core/platform/profile_utils/cpu_utils.cc:114] CPU Frequency: 2900000000 Hz
-    Get global model for round: 0
-    pull_models completed. Status:True rank:0
-    Epoch 1/2
-    1875/1875 [==============================] - 5s 2ms/step - loss: 0.0128 - accuracy: 0.9959 - val_loss: 0.1210 - val_accuracy: 0.9822
-    Epoch 2/2
-    1875/1875 [==============================] - 4s 2ms/step - loss: 0.0122 - accuracy: 0.9959 - val_loss: 0.1257 - val_accuracy: 0.9821
-    Send model to server.
-    2021-07-02 14:27:23,849 - FederatedClient - INFO - Starting to push model.
-    2021-07-02 14:27:23,853 - Communicator - INFO - Send example_project at round 0
-
-Once the fl run is complete and the server has successfully aggregrated the clients' results after all the rounds, 
-run the following commands in the fl_admin to shutdown the system (while inputting ``admin`` when prompted with user name):
-
-.. code-block:: shell
-
-    > shutdown client
-    > shutdown server
-    > bye
-
-In order to stop all processes, run ``./stop_fl.sh``.
-
-All artifacts from the FL run can be found in the server run folder you created with ``set_run_number``.  In this exercise,
-the folder is ``run_1``.
-
-Congratulations!
-You've successfully built and run a federated learning system using TensorFlow 2.
-The full `source code <https://gitlab-master.nvidia.com/dlmed/hello_nvflare/-/blob/main/examples/hello-tf2/>`_ for this exercise can be found in ``examples/hello-tf2``.
diff --git a/doc/source/index.rst b/doc/source/index.rst
deleted file mode 100644
index b6077f54fd..0000000000
--- a/doc/source/index.rst
+++ /dev/null
@@ -1,39 +0,0 @@
-:github_url: https://github.com/NVIDIA/NVFlare
-
-.. NVFlare documentation main file, created by
-   sphinx-quickstart on Wed Feb  5 09:40:29 2020.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-NVFlare
-=======
-
-
-*NVIDIA Federated Learning Application Runtime Environment*
-
-
-Getting started
----------------
-
-.. toctree::
-   :maxdepth: 1
-   :caption: User Guide
-
-   hello_pt
-   hello_numpy
-   hello_tf2
-   
-
-.. toctree::
-  :maxdepth: 1
-  :caption: Installation
-
-  installation
-
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
diff --git a/docs/_static/css/additions.css b/docs/_static/css/additions.css
new file mode 100644
index 0000000000..999ff74614
--- /dev/null
+++ b/docs/_static/css/additions.css
@@ -0,0 +1,3 @@
+.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a{display:block;background:#b1b1b1;padding:.4045em 7.3em}
+.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a{display:block;background:#a9a9a9;padding:.4045em 8.8em}
+.wy-menu-vertical li.toctree-l5{font-size: .9em;}
\ No newline at end of file
diff --git a/docs/apidocs/modules.rst b/docs/apidocs/modules.rst
new file mode 100644
index 0000000000..18162c0b7a
--- /dev/null
+++ b/docs/apidocs/modules.rst
@@ -0,0 +1,7 @@
+nvflare
+=======
+
+.. toctree::
+   :maxdepth: 4
+
+   nvflare
diff --git a/docs/appendix.rst b/docs/appendix.rst
new file mode 100644
index 0000000000..e9d55d916c
--- /dev/null
+++ b/docs/appendix.rst
@@ -0,0 +1,10 @@
+########
+Appendix
+########
+
+
+.. toctree::
+   :maxdepth: 1
+
+   appendix/create_certificate
+   appendix/FLAdminAPI
diff --git a/docs/appendix/create_certificate.rst b/docs/appendix/create_certificate.rst
new file mode 100644
index 0000000000..dbce6cc311
--- /dev/null
+++ b/docs/appendix/create_certificate.rst
@@ -0,0 +1,46 @@
+#######################################################################################
+How to create the self-signed SSL Certificate Authority and server-clients certificates
+#######################################################################################
+
+If you are using the automatically generated startup kits created by the provisioning tool as described in the
+:ref:`user_guide/provisioning_tool:Provisioning in NVIDIA FLARE`, the following would have already been taken care of at the
+creation of the startup kits. The code that generates the startup kits automatically creates certificates for the server
+and clients and puts them all in their default expected locations.
+
+.. highlight:: none
+
+If you are not using the startup kits and want to create certificates yourself manually, the following is an example of
+how that could be done::
+
+    ## 1. Server root key and certificate
+
+    ## 1.1 Server creates the root private key
+    `openssl genrsa -out rootCA.key 2048`   Or
+    `openssl genrsa -des3 -out rootCA.key 2048` (with password)
+
+    ## 1.2 Server creates the self-signed root certificate
+    `openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 -out rootCA.pem`
+
+    ## 2. Server private key and CSR
+
+    ## 2.1 Server creates private key
+    `openssl genrsa -out server.key 2048`
+    ## 2.2 Server creates certificate signing request (CSR)
+    `openssl req -new -key server.key -out server.csr`
+    ## 2.3 Server signs the CSR using the root certificate rootCA.pem
+    `openssl x509 -req -in server.csr -CA rootCA.pem -CAkey rootCA.key -CAcreateserial -out server.crt -days 500 -sha256`
+
+    ## 3. Client certificate
+
+    ## Important: client must input a common name on command below
+    ## 3.1 Client creates private key
+    `openssl genrsa -out client3.key 2048`
+    ## 3.2 Client creates CSR
+    `openssl req -new -key client3.key -out client3.csr`
+
+    ## 4. Sign the CSR using the root certificate and place signed certificate in client's config path
+
+    ## 4.1 Server runs this after getting client3.csr
+    `openssl x509 -req -in client3.csr -CA rootCA.pem -CAkey rootCA.key -CAcreateserial -out client3.crt -days 500 -sha256`
+
+    ## 4.2 Server gives client client3.crt to place in the client's config path
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000000..d363f05657
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,121 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import sphinx_rtd_theme
+import os
+import sys
+from sphinx.domains.python import PythonDomain
+
+
+class PatchedPythonDomain(PythonDomain):
+    def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode):
+        if "refspecific" in node:
+            del node["refspecific"]
+        return super(PatchedPythonDomain, self).resolve_xref(
+            env, fromdocname, builder, typ, target, node, contnode)
+
+
+sys.path.insert(0, os.path.abspath(".."))
+print(sys.path)
+
+# -- Project information -----------------------------------------------------
+
+project = "NVIDIA FLARE"
+copyright = "2021, NVIDIA"
+author = "NVIDIA"
+
+# The full version, including alpha/beta/rc tags
+release = "2.0"
+version = "2.0"
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+# Add napoleon to the extensions list
+# source_parsers = {'.md': CommonMarkParser}
+
+templates_path = ["templates"]
+
+source_suffix = {
+    ".rst": "restructuredtext",
+    ".txt": "restructuredtext",
+    ".md": "markdown",
+}
+
+extensions = [
+    "recommonmark",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.autosectionlabel",
+    "sphinxcontrib.exceltable",
+]
+
+autoclass_content = "both"
+add_module_names = False
+autosectionlabel_prefix_document = True
+
+# Add any paths that contain templates here, relative to this directory.
+# templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+html_theme_options = {
+    "collapse_navigation": True,
+    "display_version": True,
+    "navigation_depth": 5,
+    "sticky_navigation": True,  # Set to False to disable the sticky nav while scrolling.
+    # 'logo_only': True,  # if we have a html_logo below, this shows /only/ the logo with no title text
+}
+html_scaled_image_link = False
+html_show_sourcelink = True
+html_favicon = "favicon.ico"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+
+def setup(app):
+    app.add_domain(PatchedPythonDomain, override=True)
+    app.add_css_file("css/additions.css")
diff --git a/docs/examples/hello_cross_val.rst b/docs/examples/hello_cross_val.rst
new file mode 100644
index 0000000000..6f7e7c98d9
--- /dev/null
+++ b/docs/examples/hello_cross_val.rst
@@ -0,0 +1,348 @@
+Quickstart (Numpy - Cross Site Validation)
+==========================================
+
+Before You Start
+----------------
+
+Before jumping into this QuickStart guide, make sure you have an environment with `NVIDIA FLARE <https://pypi.org/project/nvflare/>`_ installed.  You can follow the
+:doc:`installation <../installation>` guide on the general concept of Python virtual environment (the recommended environment) and how to
+install NVIDIA FLARE.
+
+Prerequisite
+-------------
+
+This example builds on the :doc:`Hello Numpy <hello_numpy>` example. Please make sure you go through it completely as the concepts are heavily tied.
+
+Introduction
+-------------
+
+This tutorial is meant to solely demonstrate how the NVIDIA FLARE system works, without introducing any actual deep learning concepts.
+Through this exercise, you will learn how to use NVIDIA FLARE with numpy to perform cross site validation after training. The training process is explained in the :doc:`Hello Numpy <hello_numpy>` example.
+Using simplified weights and metrics, you will be able to clearly see how NVIDIA FLARE performs validation across different sites with little extra work.
+
+The design of this exercise follows on the :doc:`Hello Numpy <hello_numpy>` example which consists of one **server** and two **clients**, starting with weights ``[0, 1]``. For the purpose of this basic example, we'll consider the Fibonacci array as our local model. Cross site validation consists of the following steps:
+
+During training: 
+
+#. Trainer must save a local model to disk.
+#. Trainer must register this local model for validation using MLModelRegistry (explained later).
+#. Server must register any models during training (explained later).
+
+Cross site validation:
+
+#. Client retrieves the local model from MLModelRegistry and submits it to the server.
+#. Client then asks the server for models to validate.
+#. Server finds the list of models this client should validate.
+#. Server sends one of these models to the client along with a flag *more_models_available*.
+#. Client validates the received model from server and submits the results.
+#. Client checks the *more_models_available* flag and decides if another request should be sent or cross validation should be finished.
+
+During this exercise, we will see how NVIDIA FLARE takes care of most of the above steps with little work from the user. We will be working with the ``hello-cross-val`` application in the examples folder. Every custom FL application must contain three folders:
+
+ #. **custom**: contains the custom components (``trainer.py``, ``model_persistor.py``, ``validator.py``)
+ #. **config**: contains client and server configurations (``config_fed_client.json``, ``config_fed_server.json``)
+ #. **resources**: contains the logger config (``log.config``)
+
+Let's get started. First clone the repo, if you haven't already:
+
+.. code-block:: shell
+
+  $ git clone https://github.com/nvidia/nvflare.git
+
+Remember to activate your NVIDIA FLARE Python virtual environment from the installation guide. Ensure numpy is installed.
+
+.. code-block:: shell
+
+  (nvflare-env) $ python3 -m pip install numpy
+
+Now that you have all your dependencies installed, let's implement the Federated Learning system.
+
+
+Register MLModel during training
+--------------------------------
+ 
+In the :doc:`Hello Numpy <hello_numpy>` example, we implemented the ``Trainer`` object that calculates a 
+new sequence to submit to the server. For this example, we will consider this new sequence as this 
+client's local model. In order to get our model validated on other client's data, first, we must register
+it during training.
+
+.. literalinclude:: ../../hello_nvflare/examples/hello-cross-val/custom/trainer.py
+   :language: python
+   :lines: 47-62 
+   :caption: trainer.py
+
+Let's take a look at the above code in *trainer.py*. First, we must save a model to disk so that it 
+can be used later. In this example, we simply save the numpy array as a *.npz* file. In a real application, 
+this could be a ML model.
+
+Then, we need to register this model for cross site validation. The *fl_ctx* contains the ``MLModelRegistry``.
+This is a shared register for keeping track of the model between training and validation. Training produces the 
+model and adds to the register while validation retrieves the model. An entry in the register is denoted by ``MLModelEntry``
+which consists of name, meta dict and files dict. Validation will look for a model entry with key ``MLModelKeys.LOCAL_BEST_MODEL``. 
+As you can see in the example, trainer registers the numpy array it saved using the **LOCAL_BEST_MODEL** key.
+
+You can learn more about ``Shareable`` and ``FLContext`` in the :doc:`hello_pt` exercise or in the :ref:`documentation <programming_guide:Key Objects>`.
+
+Note that the server also produces a global model. We can register it for cross site validation similar 
+to client. This is done in *model_persistor.py* as this is the component responsible for saving the model.
+
+.. literalinclude:: ../../hello_nvflare/examples/hello-cross-val/custom/model_persistor.py
+   :language: python
+   :lines: 65-71
+   :caption: model_persistor.py
+
+
+Implementing the Validator
+--------------------------
+
+The validator is responsible for validating the models received from the server. These models could 
+be from other clients or models generated on server. 
+
+.. literalinclude:: ../../hello_nvflare/examples/hello-cross-val/custom/validator.py
+   :language: python
+   :linenos:
+   :caption: validator.py
+
+The validator implements the **validate** function which receives a shareable. The shareable 
+contains two important keys:
+
+#. **ShareableKey.DATA**: This contains the dict of files for a model. These are the same files that
+#.  were added to MLModelEntry with ``add_files()``.
+#. **ShareableKey.META**: This contains the meta data for a model. This is the same metadata added 
+#. to the MLModelEntry with ``add_meta_data()``.
+
+To validate the model, the common steps are :
+
+#. Retreive the model files from shareable (using **ShareableKey.DATA**)
+#. Load the model from the files (user implementation)
+#. Validate the model (user implementation)
+#. Return the results in a shareable to server. 
+
+The shareable that needs to be returned must be of **TYPE_METRICS**. Metrics are added using 
+**ShareableKey.METRICS** key.
+
+.. note::
+
+  Note that in our hello-examples, we are demonstrating Federated Learning using the familiar deep learning model concept.
+
+  However, Federated Learning is not restricted to just deep learning and may not always involve models. Thus, 
+  we define a :ref:`Learnable <programming_guide:Learnable>` object (subclasses ``dict``) as the most general form of data object generated through Federated Learning.
+  The ``Model`` class we are using here is simply a subclass of ``Learnable``.
+
+
+Application Configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Inside the config folder there are two files, ``config_fed_client.json`` and ``config_fed_server.json``.
+By default, cross site validation is disabled. In order to enable it, the following configuration needs to be added
+
+.. literalinclude:: ../../hello_nvflare/examples/hello-cross-val/config/config_fed_client.json
+   :language: python
+   :lines: 13-22
+   :caption: config_fed_client.json
+
+The key settings are:
+
+#. **cross_validate**: This flag turns cross validation workflow ON or OFF. The workflow includes sharing own local model & validating other models.
+#. **cross_site_validation**: This configuration defines the workflow parameters.
+#. **is_participating**: If True, client shares their local model with server. If false, client does not share their model.
+#. **validator**: Defines the validator to run validation.
+
+.. note::
+
+    Note that if cross site validation workflow is enabled, the client will always validate models from other sources.
+
+Cross site validation!
+----------------------
+
+Now you must set up a local environment and generate packages to simulate the server, clients, and admin. The steps to set up and run this application are identical to the :doc:`Hello Numpy <hello_numpy>` example except the app is now ``hello-cross-val`` instead of ``hello-numpy``.
+
+Setting Up the Application Environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+This command generates a poc folder with server, one client and one admin:
+
+.. code-block:: shell
+
+    $ poc
+
+Here we duplicate the client folder into two folders to create two clients, site-1 and site-2:
+
+.. code-block:: shell
+
+    $ cp -r poc/client poc/site-1
+    $ cp -r poc/client poc/site-2
+
+Finally, we copy necessary files (the exercise codes) to a working folder:
+
+.. code-block:: shell
+
+  $ mkdir -p poc/admin/transfer
+  $ cp -rf examples/* poc/admin/transfer
+
+With both the client and server ready, you can now run everything and see federated
+learning in action. FL systems usually have a server and multiple clients. We
+therefore have to start the server first:
+
+.. code-block:: shell
+
+    $ ./poc/server/startup/start.sh
+
+Once the server is running you can start the clients in different terminals.
+Open a new terminal and start the first client:
+
+.. code-block:: shell
+
+    $ ./poc/site-1/startup/start.sh site-1 localhost
+
+Open another terminal and start the second client:
+
+.. code-block:: shell
+
+    $ ./poc/site-2/startup/start.sh site-2 localhost
+
+In one last terminal, start the admin:
+
+.. code-block:: shell
+
+  $ ./poc/admin/startup/fl_admin.sh localhost
+
+
+This will launch a command prompt, where you can input commands to control and monitor many aspects of
+the FL process. Log in by entering ``admin`` for both the username and password.
+
+Running the FL
+^^^^^^^^^^^^^^
+
+Enter the commands below in order.  Pay close attention to what happens in each of four terminals.  You
+can see the admin controls server and clients with each command.
+
+.. code-block:: shell
+
+    > upload_app hello-cross-val
+
+Uploads an application in the server's registry.  This creates the application entry, populates the configuration and links the name
+``hello-cross-val`` with such application configuration.  Later, you can control this application via this name.
+
+.. code-block:: shell
+
+    > set_run_number 1
+
+Creates a workspace with the run_number on the server and all clients.  The purpose of this workspace is to isolate different runs so
+the information in one particular run does not interfere with other runs.
+
+.. code-block:: shell
+
+    > deploy_app hello-cross-val all
+
+This will make the hello-cross-val application the active one in the run_number workspace.  After the above two commands, the
+server and all the clients know the hello-cross-val application will reside in the ``run_1`` workspace.
+
+
+.. code-block:: shell
+
+    > start_app all
+
+This ``start_app`` command instructs the NVIDIA FLARE server and clients to start training with the hello-cross-val application in that ``run_1`` workspace.
+
+From time to time, you can issue ``check_status server`` in the admin client to check the entire training progress. During the first phase,
+the model will be trained. During the second phase, cross site validation will happen. The status of each
+client will change to cross site validation as it enters this second phase.
+
+Accessing the results
+---------------------
+
+During cross site validation, every client validates other clients' models and server models (if present).
+This can produce a lot of results. All the results are kept on the server in 
+*<run_dir>/cross_val_results.json*. All the models sent to the server are also present in the
+*<run_dir>/<client_uid>/* directory. To access results using fl_admin, please run:
+
+.. code-block:: shell
+
+    > validate <data_client> <model_client>
+
+OR
+
+.. code-block:: shell
+
+    > validate all
+
+The results will be in the json format.
+
+
+Understanding the Output
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+After starting the server and clients, you should begin to see 
+some outputs in each terminal tracking the progress of the FL run. 
+As each client finishes training, it will start the cross site validation process.
+Druing this you'll see several important outputs the track the progress 
+of cross site validation.
+
+Let's see the progress of cross site validation using output snippets. When 
+cross site validation begins, you should see the following prompt on client side.
+This mentions the timeout (max time to wait without receiving models from server) 
+and the response of server for submitting client's own local model.
+
+.. code-block:: shell
+    :caption: site-1
+
+    2021-08-29 15:05:32,312 - CrossSiteValManager - INFO - Cross validation timeout is set to 10 minutes.
+    2021-08-29 15:05:32,351 - Communicator - INFO - Server reply to SubmitBestLocalModel:  Received best model from site-1.. SubmitBestLocalModel time: 0.014548540115356445 seconds
+
+Now the client will start asking server for models. One scenario could be that 
+server has no models currently available. In that case, server will ask client to wait
+until more models are available. The output will look like 
+
+.. code-block:: shell
+    :caption: site-2
+
+    2021-08-29 15:05:32,364 - FederatedClient - INFO - Getting other models from server for cross validation.
+    2021-08-29 15:05:32,377 - Communicator - INFO - Received 0 models for validation. GetValidationModels time: 0.012315750122070312 seconds
+    2021-08-29 15:05:32,382 - FederatedClient - INFO - Server has no models available currently. Waiting 60 secs before asking again.
+
+Another scenario is that the server has models available. In this case, client will receive the model and
+send it to validator. After validation, the results will be sent back to the server. Client output 
+will look something like
+
+.. code-block:: shell
+    :caption: server
+
+    2021-08-29 15:06:32,451 - FederatedClient - INFO - Getting other models from server for cross validation.
+    2021-08-29 15:06:32,468 - Communicator - INFO - Received 1 models for validation. GetValidationModels time: 0.014800786972045898 seconds
+    2021-08-29 15:06:32,495 - SimpleValidator - INFO - Numpy model loaded from shareable: [54. 88.]
+    2021-08-29 15:06:32,495 - SimpleValidator - INFO - Metadata received with model: {'cookie': {'cross_val_model_owner': 'site-1'}}
+    2021-08-29 15:06:32,496 - SimpleValidator - INFO - Validating the model.
+    2021-08-29 15:06:32,497 - FederatedClient - INFO - Submitting cross validation results to server.
+    2021-08-29 15:06:32,513 - Communicator - INFO - Received comments:  Received Cross Validation results from site-1.. SubmitCrossSiteValidationResults time: 0.014247655868530273 seconds
+
+As you can see, the client receives a shareable with Numpy array of ``[54, 88]`` and meta data showing the owner of the model.
+After validation the client sends the results to the server.
+
+On the server side, you can see a full log of each client who requested the model and the results received.
+
+.. code-block:: shell
+    :caption: server
+
+    2021-08-29 15:52:47,670 - ServerCrossSiteValManager - INFO - Received best model from site-2
+    2021-08-29 15:52:47,695 - ServerCrossSiteValManager - INFO - Client site-2 requested models for cross site validation.
+    2021-08-29 15:52:47,696 - ServerCrossSiteValManager - INFO - Sent 1 out of 2 models to site-2. Will be asked to wait and retry.
+    2021-08-29 15:52:47,697 - ServerCrossSiteValManager - INFO - Sending site-1's model to site-2 for validation.
+    2021-08-29 15:52:47,737 - ServerCrossSiteValManager - INFO - site-2 submitted results of validating site-1's model.
+    2021-08-29 15:52:47,738 - ServerCrossSiteValManager - INFO - New results added are: {'site-2': {'site-1': {'result': [10, 11]}}}
+
+As you can see the server shows the log of each client requesting models, the models it sends and the 
+results received. Since the server could be responding to many clients at the same time, it may 
+require careful examination to make proper sense of events from the jumbled logs.
+
+Once the FL run is complete and the server has successfully aggregrated the client's results after all the rounds, and cross site validation is finished, run the following commands in the fl_admin to shutdown the system (while inputting ``admin`` when prompted with password):
+
+.. code-block:: shell
+
+    > shutdown client
+    > shutdown server
+    > bye
+
+In order to stop all processes, run ``./stop_fl.sh``. 
+
+Congratulations!
+You've successfully run your numpy federated learning system with cross site validation.
diff --git a/docs/examples/hello_numpy.rst b/docs/examples/hello_numpy.rst
new file mode 100644
index 0000000000..e47f3f3352
--- /dev/null
+++ b/docs/examples/hello_numpy.rst
@@ -0,0 +1,238 @@
+Quickstart (Numpy)
+===================
+
+Before You Start
+----------------
+
+Before jumping into this QuickStart guide, make sure you have an environment with `NVIDIA FLARE <https://pypi.org/project/nvflare/>`_
+installed. You can follow :doc:`installation <../installation>` on the general concept of setting up a Python virtual
+environment (the recommended environment) and how to install NVIDIA FLARE.
+
+
+Introduction
+-------------
+
+This tutorial is meant to solely demonstrate how the NVIDIA FLARE system works, without introducing any actual deep
+learning concepts. Through this exercise, you will learn how to use NVIDIA FLARE with numpy to perform basic
+computations across two clients with the included Scatter and Gather workflow, which broadcasts the training tasks then
+aggregates the results that come back. Due to the simplified weights, you will be able to clearly see and understand
+the results of the FL aggregation and the model persistor process.
+
+The design of this exercise consists of one **server** and two **clients**, starting with weights ``[[1, 2, 3], [4, 5, 6], [7, 8, 9]]``.
+The following steps compose one cycle of weight updates, called a **round**:
+
+ #. Clients are responsible for adding a delta to the weights to calculate new weights for the model.
+ #. These updates are then sent to the server which will aggregate them to produce a model with new weights.
+ #. Finally, the server sends this updated version of the model back to each client, so the clients can continue to calculate the next model weights in future rounds.
+
+For this exercise, we will be working with the ``hello-numpy-fed-avg`` application in the examples folder.
+Every custom FL application must contain three folders:
+
+ #. **custom**: contains the custom components (``np_trainer.py``, ``np_model_persistor.py``)
+ #. **config**: contains client and server configurations (``config_fed_client.json``, ``config_fed_server.json``)
+ #. **resources**: contains the logger config (``log.config``)
+
+Let's get started. First clone the repo, if you haven't already:
+
+.. code-block:: shell
+
+  $ git clone https://github.com/nvidia/nvflare.git
+
+Remember to activate your NVIDIA FLARE Python virtual environment from the installation guide. Ensure numpy is installed.
+
+.. code-block:: shell
+
+  (nvflare-env) $ python3 -m pip install numpy
+
+Now that you have all your dependencies installed, let's implement the federated learning system.
+
+
+NVIDIA FLARE Client
+-------------------
+ 
+In a file called ``np_trainer.py``, import nvflare and numpy.
+Now we will implement the ``execute`` function to enable the clients to perform
+a simple addition of a diff to represent one calculation of training a round.
+
+See :ref:`hello_numpy_np_trainer` or find the full code of ``np_trainer.py`` at
+``examples/hello-numpy-fed-avg/custom/np_trainer.py`` to follow along.
+
+The server sends either the initial weights or any stored weights to each of the clients
+through the ``Shareable`` object passed into ``execute()``. Each client adds the
+diff to the model data after retrieving it from the DXO (see :ref:`programming_guide/data_exchange_object`)
+obtained from the Shareable, and creates a new ``Shareable`` to include the new weights also contained
+within a DXO.
+
+In a real federated learning training scenario, each client does its training independently on its own dataset. 
+As such, the weights returned to the server will likely be different for each of the clients. 
+
+The FL server can ``aggregrate`` (in this case average) the clients' results to produce the aggregated model.
+
+You can learn more about ``Shareable`` and ``FLContext`` in the :ref:`documentation <programming_guide>`.
+
+
+NVIDIA FLARE Server & Application
+---------------------------------
+
+Model Persistor
+^^^^^^^^^^^^^^^^^
+
+.. literalinclude:: ../../examples/hello-numpy-fed-avg/custom/np_model_persistor.py
+   :language: python
+   :linenos:
+   :caption: np_model_persistor.py
+
+The model persistor is used to load and save models on the server. Here, the model is weights packaged into a ``ModelLearnable`` object.
+
+Internally, DXO is used to manage data after :class:`FullModelShareableGenerator<nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator>`
+converts Learnable to Shareable on the FL server. The DXO helps all of the FL components agree on the format.
+
+In this exercise, we can simply save the model as a binary ".npy" file.
+Depending on the frameworks and tools, the methods of saving the model may vary.
+
+Application Configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Inside the config folder there are two files, ``config_fed_client.json`` and ``config_fed_server.json``.
+For now, the default configurations are sufficient.
+
+
+.. literalinclude:: ../../examples/hello-numpy-fed-avg/config/config_fed_server.json
+   :language: json
+   :linenos:
+   :caption: config_fed_server.json
+
+
+Note that the component with id ``persistor`` points to the custom ``NPModelPersistor`` with full Python module path.
+
+
+.. literalinclude:: ../../examples/hello-numpy-fed-avg/config/config_fed_client.json
+   :language: json
+   :linenos:
+   :caption: config_fed_client.json
+
+
+Here, in ``executors``, the Trainer implementation ``NPTrainer`` is configured for the task "train".
+
+
+Federated Numpy with Scatter and Gather Workflow!
+-------------------------------------------------
+
+Now you must set up a local environment and generate packages to simulate the server, clients, and admin.
+
+Setting Up the Application Environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+This command generates a poc folder with server, one client and one admin:
+
+.. code-block:: shell
+
+    $ poc
+
+Here we duplicate the client folder into two folders to create two clients, site-1 and site-2:
+
+.. code-block:: shell
+
+    $ cp -r poc/client poc/site-1
+    $ cp -r poc/client poc/site-2
+
+Finally, we copy necessary files (the exercise codes) to a working folder:
+
+.. code-block:: shell
+
+  $ mkdir -p poc/admin/transfer
+  $ cp -rf examples/* poc/admin/transfer
+
+With both the client and server ready, you can now run everything and see federated
+learning in action. FL systems usually have a server and multiple clients. We
+therefore have to start the server first:
+
+.. code-block:: shell
+
+    $ ./poc/server/startup/start.sh
+
+Once the server is running you can start the clients in different terminals.
+Open a new terminal and start the first client:
+
+.. code-block:: shell
+
+    $ ./poc/site-1/startup/start.sh site-1 localhost
+
+Open another terminal and start the second client:
+
+.. code-block:: shell
+
+    $ ./poc/site-2/startup/start.sh site-2 localhost
+
+In one last terminal, start the admin:
+
+.. code-block:: shell
+
+  $ ./poc/admin/startup/fl_admin.sh localhost
+
+
+This will launch a command prompt, where you can input commands to control and monitor many aspects of
+the FL process. Log in by entering ``admin`` for both the username and password.
+
+Running the FL
+^^^^^^^^^^^^^^
+
+Enter the commands below in order.  Pay close attention to what happens in each of four terminals.  You
+can see the admin controls server and clients with each command.
+
+.. code-block:: shell
+
+    > upload_app hello-numpy-fed-avg
+
+Uploads the application from the admin client to the server's staging area.
+
+.. code-block:: shell
+
+    > set_run_number 1
+
+Creates a run directory in the workspace for the run_number on the server and all clients. The run directory allows for
+the isolation of different runs so the information in one particular run does not interfere with other runs.
+
+.. code-block:: shell
+
+    > deploy_app hello-numpy-fed-avg all
+
+This will make the hello-numpy-fed-avg application the active one in the run_number workspace.  After the above two commands, the
+server and all the clients know the hello-numpy-fed-avg application will reside in the ``run_1`` workspace.
+
+
+.. code-block:: shell
+
+    > start_app all
+
+This ``start_app`` command instructs the NVIDIA FLARE server and clients to start training with the hello-numpy-fed-avg
+application in the ``run_1`` workspace.
+
+From time to time, you can issue ``check_status server`` in the admin client to check the entire training progress.
+
+You should now see how the training does in the very first terminal (the one that started the server):
+
+
+Understanding the Output
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+After starting the server and clients, you should begin to see 
+some outputs in each terminal tracking the progress of the FL run. If everything went as
+planned, you should see that through 10 rounds, the FL system has aggregated new models on the server
+with the results produced by the clients.
+
+Once the fl run is complete and the server has successfully aggregated the client's results after all the rounds, run
+the following commands in the fl_admin to shutdown the system (while inputting ``admin`` when prompted with password):
+
+.. code-block:: shell
+
+    > shutdown client
+    > shutdown server
+    > bye
+
+In order to stop all processes, run ``./stop_fl.sh``. 
+
+If you want to reset the saved server weights to start from ``[0,1]`` again, delete ``weights.pickle`` in the ``server/run_1/mmar_server/models/`` directory.
+
+Congratulations!
+You've successfully built and run your first numpy federated learning system. 
+You now have a decent grasp of the main FL concepts, and are ready to start exploring how NVIDIA FLARE can be applied to many other tasks.
diff --git a/docs/examples/hello_numpy_np_trainer.rst b/docs/examples/hello_numpy_np_trainer.rst
new file mode 100644
index 0000000000..7dc5905d8f
--- /dev/null
+++ b/docs/examples/hello_numpy_np_trainer.rst
@@ -0,0 +1,7 @@
+Code of np_trainer
+------------------
+
+.. literalinclude:: ../../examples/hello-numpy-fed-avg/custom/np_trainer.py
+   :language: python
+   :linenos:
+   :caption: np_trainer.py
diff --git a/doc/source/hello_pt.rst b/docs/examples/hello_pt.rst
similarity index 81%
rename from doc/source/hello_pt.rst
rename to docs/examples/hello_pt.rst
index db505695e3..4cc5f95a10 100644
--- a/doc/source/hello_pt.rst
+++ b/docs/examples/hello_pt.rst
@@ -4,18 +4,18 @@ Quickstart (PyTorch)
 Before You Start
 ----------------
 
-Feel free to refer to the official :doc:`documentation <../programming_guide>` at any point to learn more about the specifics of `NVFlare <https://pypi.org/project/nvflare/>`_.
+Feel free to refer to the official :doc:`documentation <../programming_guide>` at any point to learn more about the specifics of `NVIDIA FLARE <https://pypi.org/project/nvflare/>`_.
 
-Make sure you have an environment with NVFlare installed.  You can follow the
-:doc:`hello_installation` guide on the general concept of Python virtual environment (the recommended environment) and how to
-install NVFlare.
+Make sure you have an environment with NVIDIA FLARE installed.  You can follow the
+:doc:`installation <../installation>` guide on the general concept of Python virtual environment (the recommended environment) and how to
+install NVIDIA FLARE.
 
 
 Introduction
 -------------
 
-Through this exercise, you will integrate NVFlare with the popular
-deep learning framework `PyTorch <https://pytorch.org/>`_ and learn how to use NVFlare to train a convolutional network with the CIFAR10 dataset.  
+Through this exercise, you will integrate NVIDIA FLARE with the popular
+deep learning framework `PyTorch <https://pytorch.org/>`_ and learn how to use NVIDIA FLARE to train a convolutional network with the CIFAR10 dataset.
 
 The design of this exercise consists of one **server** and two **clients** all having the same PyTorch model. 
 The following steps compose one cycle of weight updates, called a **round**:
@@ -37,7 +37,7 @@ Now that you have a rough idea of what is going on, let's get started. First clo
 
   $ git clone https://github.com/nvidia/nvflare.git
 
-Now remember to activate your NVFlare Python virtual environment from the installation guide.
+Now remember to activate your NVIDIA FLARE Python virtual environment from the installation guide.
 
 Since you will use PyTorch and torchvision for this exercise, let's go ahead and install both libraries: 
 
@@ -57,11 +57,11 @@ Since you will use PyTorch and torchvision for this exercise, let's go ahead and
 
 If you would like to go ahead and run the exercise now, you can skip directly to :ref:`hands-on`.
 
-NVFlare Client
----------------
+NVIDIA FLARE Client
+-------------------
 
-Network
-^^^^^^^^
+Neural Network
+^^^^^^^^^^^^^^^
 
 With all the required dependencies installed, you are ready to run a Federated Learning
 with two clients and one server. The training procedure and network 
@@ -71,28 +71,29 @@ architecture are modified from
 
 Let's see what an extremely simplified CIFAR10 training looks like:
 
-.. literalinclude:: ../../hello_nvflare/examples/hello-pt/custom/net.py
+.. literalinclude:: ../../examples/hello-pt/custom/pt_net.py
    :language: python
-   :caption: net.py
+   :caption: pt_net.py
 
 This ``Net`` class is your convolutional neural network to train with the CIFAR10 dataset.
-This is not related to NVFlare, so we implement it in a file called ``net.py``.
+This is not related to NVIDIA FLARE, so we implement it in a file called ``net.py``.
 
 Dataset & Setup
 ^^^^^^^^^^^^^^^^
 
-Now all we have left is to implement the one required custom class related to NVFlare, ``Trainer``, in a file called ``trainer.py``.
+Now all we have left is to implement the one required custom class related to NVIDIA FLARE, ``SimpleTrainer``, in a file
+called ``trainer.py``.
 
-In a real FL, each client would have their own dataset used for their local training.
+In a real FL experiment, each client would have their own dataset used for their local training.
 For simplicity's sake, you can download the same CIFAR10 dataset from the Internet via torchvision's datasets module.
 Additionally, you need to setup the optimizer, loss function and transform to process the data.
-You can think of all of this code as part of your local train, as every deep learning training has a similar setup.
+You can think of all of this code as part of your local training loop, as every deep learning training has a similar setup.
 
-Since you will encapsulate every training-related step in the ``Trainer`` class, let's put this preparation stage into the ``__init__`` method:
+Since you will encapsulate every training-related step in the ``SimpleTrainer`` class, let's put this preparation stage into the ``__init__`` method:
 
-.. literalinclude:: ../../hello_nvflare/examples/hello-pt/custom/trainer.py
+.. literalinclude:: ../../examples/hello-pt/custom/trainer.py
    :language: python
-   :lines: 1-30
+   :lines: 1-31
 
 
 Local Train
@@ -110,18 +111,24 @@ The number of epochs is hardcoded to 2, but this can be easily set from a parame
 
 .. note::
 
-  Everything up to this point is completely independent of NVFlare. It is just purely a PyTorch
-  deep learning exercise.  You will now build the NVFlare application based on this PyTorch code.
+  Everything up to this point is completely independent of NVIDIA FLARE. It is just purely a PyTorch
+  deep learning exercise.  You will now build the NVIDIA FLARE application based on this PyTorch code.
 
 
-Integrate NVFlare with Local Train
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Integrate NVIDIA FLARE with Local Train
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-NVFlare makes it very easy to integrate your local train code into the NVFlare API.
+NVIDIA FLARE makes it very easy to integrate your local train code into the NVIDIA FLARE API.
 
 The simplest way to do this is to subclass the ``Trainer`` class and
 implement one method ``train``, which is called every time the client receives 
-an updated model from the server.  We can then call our local train inside the ``train`` method.
+an updated model from the server. We can then call our local train inside the ``train`` method.
+
+.. note::
+
+  The ``train`` method inside the ``Trainer`` class is where all of the client side computation occurs.
+  In these exercises, we update the weights by training on a local dataset, however, it is important to remember that NVIDIA FLARE is not restricted to just deep learning.
+  The type of data passed between the server and the clients, and the computations that the clients perform can be anything, as long as all of the FL Components agree on the same format. 
 
 Take a look at the following code:
 
@@ -129,7 +136,7 @@ Take a look at the following code:
    :language: python
    :pyobject: SimpleTrainer.train
 
-The concept of ``Shareable`` is described briefly at :ref:`shareable` section.  Essentially, every NVFlare client receives the model weights
+The concept of ``Shareable`` is described briefly at :ref:`shareable` section.  Essentially, every NVIDIA FLARE client receives the model weights
 from the server in ``shareable`` passed into the train method, and returns a new ``shareable`` back to the server.
 
 Thus, the first thing is to retrieve the model weights delivered by server via ``shareable``:
@@ -152,7 +159,7 @@ We then perform a local train so the client's model is trained with its own data
   self.local_train()
 
 After finishing the local train, the train method builds a new ``shareable`` with newly-trained weights and metadata and returns it
-back to the NVFlare server for aggregation:
+back to the NVIDIA FLARE server for aggregation:
 
 .. code-block:: python
 
@@ -196,12 +203,12 @@ FLContext
 
 You can find more details in the :ref:`documentation <programming_guide:FLContext>`.
 
-NVFlare Server & Application
-------------------------------
+NVIDIA FLARE Server & Application
+---------------------------------
 
-In this exercise, you can use the default settings, which leverage NVFlare built-in components for NVFlare server.  These
+In this exercise, you can use the default settings, which leverage NVIDIA FLARE built-in components for NVIDIA FLARE server.  These
 built-in components are commonly used in most deep learning scenarios.  However, you are encouraged to build your own components
-to fully customize NVFlare to meet your environment, which we will demonstrate in the following exercises.
+to fully customize NVIDIA FLARE to meet your environment, which we will demonstrate in the following exercises.
 
 
 Application Configuration
@@ -214,9 +221,9 @@ Inside the config folder there are two files, ``config_fed_client.json`` and ``c
    :linenos:
    :caption: config_fed_client.json
 
-Take a look at line 9.  This is the ``SimpleTrainer`` you just implemented.  The NVFlare client loads this
+Take a look at line 9.  This is the ``SimpleTrainer`` you just implemented.  The NVIDIA FLARE client loads this
 application configuration and picks your implementation.  You can easily change it to another class so
-your NVFlare client has different training logic.
+your NVIDIA FLARE client has different training logic.
 
 
 .. literalinclude:: ../../hello_nvflare/examples/hello-pt/config/config_fed_server.json
@@ -224,11 +231,11 @@ your NVFlare client has different training logic.
    :linenos:
    :caption: config_fed_server.json
 
-The server application configuration, like said before, leverages NVFlare built-in components. Remember, you
+The server application configuration, like said before, leverages NVIDIA FLARE built-in components. Remember, you
 are encouraged to change them to your own classes whenever you have different application logic.
 
 Note that on line 26, ``persistor`` points to ``PTFileModelPersistor``. 
-NVFlare provides a built-in PyTorch implementation for a model persistor, however for other frameworks/libraries, you will have to implement your own.
+NVIDIA FLARE provides a built-in PyTorch implementation for a model persistor, however for other frameworks/libraries, you will have to implement your own.
 
 
 .. _hands-on:
@@ -325,7 +332,7 @@ the server and all the clients know the hello-pt application will reside in the
 
     > start_app all
 
-This ``start_app`` command instructs the NVFlare server and clients to start training with the hello-pt application in that ``run_1`` workspace.
+This ``start_app`` command instructs the NVIDIA FLARE server and clients to start training with the hello-pt application in that ``run_1`` workspace.
 
 From time to time, you can issue ``check_status server`` in the admin client to check the entire training progress.
 
diff --git a/docs/examples/hello_tf2.rst b/docs/examples/hello_tf2.rst
new file mode 100644
index 0000000000..1b1c78caf4
--- /dev/null
+++ b/docs/examples/hello_tf2.rst
@@ -0,0 +1,331 @@
+Quickstart (TensorFlow 2)
+===========================
+
+Before You Start
+----------------
+
+We recommend you first finish either the :doc:`hello_pt` or the :doc:`hello_numpy` exercise.
+Those guides go more in depth in explaining the federated learning aspect of `NVIDIA FLARE <https://pypi.org/project/nvflare/>`_.
+
+Here we assume you have already installed NVIDIA FLARE inside a python virtual environment and have already cloned the repo.
+
+Introduction
+-------------
+
+Through this exercise, you will integrate NVIDIA FLARE with the popular
+deep learning framework `TensorFlow 2 <https://www.tensorflow.org/>`_ and learn how to use NVIDIA FLARE to train a convolutional
+network with the MNIST dataset using the Scatter and Gather workflow.
+You will also be introduced to some new components and concepts, including filters, aggregrators, and event handlers.
+
+The design of this exercise consists of one **server** and two **clients** all having the same TensorFlow 2 model. 
+The following steps compose one cycle of weight updates, called a **round**:
+
+ #. Clients are responsible for generating individual weight-updates for the model using their own MNIST dataset. 
+ #. These updates are then sent to the server which will aggregate them to produce a model with new weights. 
+ #. Finally, the server sends this updated version of the model back to each client.
+
+For this exercise, we will be working with the ``hello-tf2`` application in the examples folder. 
+Every custom FL application must contain three folders:
+
+ #. **custom**: contains the custom components (``tf2_net.py``, ``trainer.py``, ``filter.py``, ``tf2_model_persistor.py``)
+ #. **config**: contains client and server configurations (``config_fed_client.json``, ``config_fed_server.json``)
+ #. **resources**: contains the logger config (``log.config``)
+
+Let's get started. Since this task is using TensorFlow, let's go ahead and install the library inside our virtual environment: 
+
+.. code-block:: shell
+
+  (nvflare-env) $ python3 -m pip install tensorflow
+
+
+NVIDIA FLARE Client
+-------------------
+
+Neural Network
+^^^^^^^^^^^^^^^
+
+With all the required dependencies installed, you are ready to run a Federated Learning system
+with two clients and one server. Before you start, let's see what a simplified MNIST network looks like.
+
+.. literalinclude:: ../../examples/hello-tf2/custom/tf2_net.py
+   :language: python
+   :lines: 15-
+   :lineno-start: 15
+   :linenos:
+   :caption: tf2_net.py
+
+This Net class is the convolutional neural network to train with MNIST dataset. This is not related to NVIDIA FLARE, so implement it in a file called ``tf2_net.py``.
+
+Dataset & Setup
+^^^^^^^^^^^^^^^^
+
+Now you have to implement the class `Trainer``, which is a subclass of ``Executor`` in NVIDIA FLARE, in a file called ``trainer.py``.
+
+Before you can really start a training, you need to set up your dataset.
+In this exercise, you can download it from the Internet via ``tf.keras``'s datasets module, and split it in half to create a separate dataset for each client.
+Additionally, you must setup the optimizer, loss function and transform to process the data.
+
+Since every step will be encapsulated in the ``SimpleTrainer`` class, let's put this preparation stage into one method ``setup``:
+
+.. literalinclude:: ../../examples/hello-tf2/custom/trainer.py
+   :language: python
+   :lines: 41-73
+   :lineno-start: 41
+   :linenos:
+
+
+How can you ensure this setup method is called before the client receives the model from the server?  The Trainer
+class is also a :ref:`FLComponent <programming_guide:FLComponent>`, which always receives ``Event`` whenever NVIDIA FLARE enters or leaves a certain stage.
+In this case, there is an ``Event`` called ``EventType.START_RUN`` which perfectly matches these requirements. 
+Because our trainer is a subclass of ``FLComponent``, you can implement the handler to handle the event and call the setup method:
+
+.. literalinclude:: ../../examples/hello-tf2/custom/trainer.py
+   :language: python
+   :lines: 37-39
+   :lineno-start: 37
+   :linenos:
+
+.. note::
+
+  This is a new concept you haven't learned in previous two exercises.  The concepts of ``event`` and ``handler`` are very powerful because
+  you are free to add your logic so it can run at different time and process various events.  The entire list of events fired by
+  NVIDIA FLARE is shown at :ref:`Event types <programming_guide/event_system>`.
+
+
+You have everything you need, now let's implement the last method called ``execute``, which is
+called every time the client receives an updated model from the server with the Task we will configure.
+
+
+Link NVIDIA FLARE with Local Train
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Take a look at the following code:
+
+.. literalinclude:: ../../examples/hello-tf2/custom/trainer.py
+   :language: python
+   :lines: 75-
+   :lineno-start: 75
+   :linenos:
+
+Every NVIDIA FLARE client receives the model weights from the server in the :ref:`shareable <programming_guide/shareable>`.
+This exercise uses a simple ``exclude_var`` filter, so make sure to replace the missing layer with weights from the clients' previous training round:
+
+.. literalinclude:: ../../examples/hello-tf2/custom/trainer.py
+   :language: python
+   :lines: 115-119
+   :lineno-start: 115
+   :linenos:
+
+Now update the local model with those received weights:
+
+.. literalinclude:: ../../examples/hello-tf2/custom/trainer.py
+   :language: python
+   :lines: 122
+   :lineno-start: 122
+   :linenos:
+
+Then perform a simple :code:`self.model.fit` so the client's model is trained with its own dataset:
+
+.. literalinclude:: ../../examples/hello-tf2/custom/trainer.py
+   :language: python
+   :lines: 126-131
+   :lineno-start: 126
+   :linenos:
+  
+After finishing the local train, the train method uses the newly-trained weights to build a new ``DXO`` to update the
+``Shareable`` with and then returns it back to the NVIDIA FLARE server.
+
+
+NVIDIA FLARE Server & Application
+---------------------------------
+
+Filter
+^^^^^^^ 
+
+:ref:`filter <programming_guide/filters>` can be used for additional data processing in the ``Shareable``, for both
+inbound and outbound data from the client and/or server.
+
+For this exercise, we use a basic ``exclude_var`` filter to exclude the variable/layer ``flatten`` from the task result
+as it goes outbound from the client to the server. The excluded layer is replaced with all zeros of the same shape,
+which reduces compression size and ensures that the clients' weights for this variable are not shared with the server.
+
+.. literalinclude:: ../../examples/hello-tf2/custom/filter.py
+   :language: python
+   :lines: 15-
+   :lineno-start: 15
+   :linenos:
+   :caption: filter.py
+
+The filtering procedure occurs in the one required method, process, which receives and returns a shareable.
+The parameters for what is excluded and the inbound/outbound option are all set in ``config_fed_client.json`` (shown later below) and passed in through the constructor.
+
+
+Model Aggregator
+^^^^^^^^^^^^^^^^
+
+The :ref:`model aggregator <programming_guide:Aggregator>` is used by the server to aggregate the clients' models into one model
+within the Scatter and Gather workflow.
+
+In this exercise, we perform a simple average over the two clients' weights with the :class:`AccumulateWeightedAggregator<nvflare.app_common.aggregators.accumulate_model_aggregator.AccumulateWeightedAggregator>`
+and configure for it to be used in ``config_fed_server.json`` (shown later below).
+
+Model Persistor
+^^^^^^^^^^^^^^^
+
+The model persistor is used to load and save models on the server.
+
+.. literalinclude:: ../../examples/hello-tf2/custom/tf2_model_persistor.py
+   :language: python
+   :lines: 15-
+   :lineno-start: 15
+   :linenos:
+   :caption: tf2_model_persistor.py
+
+In this exercise, we simply serialize the model weights dictionary using pickle and save it to a log directory calculated
+in initialize. The file is saved on the FL server and the weights file name is defined in ``config_fed_server.json``.
+Depending on the frameworks and tools, the methods of saving the model may vary.
+
+FLContext is used throughout these functions to provide various useful FL-related information.
+You can find more details in the :ref:`documentation <programming_guide/fl_context>`.
+
+Application Configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Finally, inside the config folder there are two files, ``config_fed_client.json`` and ``config_fed_server.json``.
+
+.. literalinclude:: ../../examples/hello-tf2/config/config_fed_server.json
+   :language: json
+   :linenos:
+   :caption: config_fed_server.json
+
+
+Note how the :class:`ScatterAndGather<nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather>` workflow is
+configured to use the included ``aggregator`` :class:`AccumulateWeightedAggregator<nvflare.app_common.aggregators.accumulate_model_aggregator.AccumulateWeightedAggregator>`
+and ``shareable_generator`` :class:`FullModelShareableGenerator<nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator>`.
+The ``persistor`` is configured to use ``TF2ModelPersistor`` in the custom directory of this hello_tf2 app with full
+Python module paths.
+
+
+.. literalinclude:: ../../examples/hello-tf2/config/config_fed_client.json
+   :language: json
+   :linenos:
+   :caption: config_fed_client.json
+
+
+Here, ``executors`` is configured with the Trainer implementation ``SimpleTrainer``.
+Also, we set up ``filter.ExcludeVars`` as a ``task_result_filters`` and pass in ``["flatten"]`` as the argument.
+Both of these are configured for the only Task that will be broadcast in the Scatter and Gather workflow, "train".
+
+Train the Model, Federated!
+---------------------------
+
+Now you must set up a local environment and generate packages to simulate the server, clients, and admin.
+
+Setting Up the Application Environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This command generates a poc folder with server, one client and one admin:
+
+.. code-block:: shell
+
+    $ poc
+
+Here we duplicate the client folder into two folders to create two clients, site-1 and site-2:
+
+.. code-block:: shell
+
+    $ cp -r poc/client poc/site-1
+    $ cp -r poc/client poc/site-2
+
+Finally, we copy necessary files (the exercise codes) to a working folder:
+
+.. code-block:: shell
+
+  $ mkdir -p poc/admin/transfer
+  $ cp -rf examples/* poc/admin/transfer
+
+With both the client and server ready, you can now run everything and see federated
+learning in action. FL systems usually have a server and multiple clients. We
+therefore have to start the server first:
+
+.. code-block:: shell
+
+    $ ./poc/server/startup/start.sh
+
+Once the server is running you can start the clients in different terminals.
+Open a new terminal and start the first client:
+
+.. code-block:: shell
+
+    $ ./poc/site-1/startup/start.sh site-1 localhost
+
+Open another terminal and start the second client:
+
+.. code-block:: shell
+
+    $ ./poc/site-2/startup/start.sh site-2 localhost
+
+In one last terminal, start the admin:
+
+.. code-block:: shell
+
+  $ ./poc/admin/startup/fl_admin.sh localhost
+
+
+This will launch a command prompt, where you can input commands to control and monitor many aspects of
+the FL process. Log in by entering ``admin`` for both the username and password.
+
+Running the FL
+^^^^^^^^^^^^^^
+
+Enter the commands below in order.  Pay close attention to what happens in each of four terminals.  You
+can see the admin controls server and clients with each command.
+
+.. code-block:: shell
+
+    > upload_app hello-tf2
+
+Uploads the application from the admin client to the server's staging area.
+
+.. code-block:: shell
+
+    > set_run_number 1
+
+Creates a run directory in the workspace for the run_number on the server and all clients. The run directory allows for
+the isolation of different runs so the information in one particular run does not interfere with other runs.
+
+.. code-block:: shell
+
+    > deploy_app hello-tf2 all
+
+This will make the hello-tf2 application the active one in the run_number workspace.  In this exercise, after the above two commands, the
+server and all the clients know the hello-tf2 application will reside in ``run_1`` workspace.
+
+
+.. code-block:: shell
+
+    > start_app all
+
+This ``start_app`` command instructs the NVIDIA FLARE server and clients to start training with the hello-tf2 application in the ``run_1`` workspace.
+
+From time to time, you can issue ``check_status server`` in the admin client to check the entire training progress.
+
+You should now see how the training does in the very first terminal (the one that started the server).
+
+Once the fl run is complete and the server has successfully aggregated the clients' results after all the rounds,
+run the following commands in the fl_admin to shutdown the system (while inputting ``admin`` when prompted with user name):
+
+.. code-block:: shell
+
+    > shutdown client
+    > shutdown server
+    > bye
+
+In order to stop all processes, run ``./stop_fl.sh``.
+
+All artifacts from the FL run can be found in the server run folder you created with ``set_run_number``.  In this exercise,
+the folder is ``run_1``.
+
+Congratulations!
+You've successfully built and run a federated learning system using TensorFlow 2.
+The full `source code <https://github.com/NVIDIA/NVFlare/tree/main/examples/hello-tf2>`_ for this exercise can be found in ``examples/hello-tf2``.
diff --git a/docs/faq.rst b/docs/faq.rst
new file mode 100644
index 0000000000..f00a3f008f
--- /dev/null
+++ b/docs/faq.rst
@@ -0,0 +1,207 @@
+.. _FAQ:
+
+###
+FAQ
+###
+
+************************
+Client related questions
+************************
+
+#. What happens if an FL client joins during the FL training?
+
+    An FL client can join the FL training any time. As long as the participating FL clients are still within the maximum
+    number of clients, the client can join. The newly joined client will get the current round of global model for training
+    and will contribute to the current global model.
+
+#. Do federated learning clients need to open any ports for the FL server to reach the FL client?
+
+    No, federated learning training does not require for FL clients to open their network for inbound traffic. The server
+    never sends uninvited requests to clients but only responds to client requests.
+
+.. _multi gpu training:
+
+#. Can a client train with multiple GPUs?
+
+    Yes, the administrator command ``start_mgpu client <gpu number> <client name>`` can be used to start training
+    on FL clients with different numbers of GPUs when starting training. But you will need to install ``torch`` for
+    PyTorch based Trainer, and ``mpi4py`` for all other trainers. It is possible for different clients to be
+    training with different numbers of GPUs.
+
+#. How do FL clients get identified?
+
+    The federated learning clients are identified by a dynamically generated FL token issued by the server during runtime.
+    When an FL client first joins an FL training, it first needs to send a login request to the FL server. During the login
+    process, the FL server and client need to exchange SSL certificates for bi-directional authentication. Once the
+    authentication is successful, the FL server sends an FL token to the client. The FL client will use this FL token to
+    identify itself for all following requests for the global model and all model updating operations.
+
+#. Can I run multiple FL clients from the same machine?
+
+    Yes. The FL clients are identified by FL token, not machine IP. Each FL client will have its own FL token as well as
+    instance name, which is the client name that must be used for issuing specific commands to that client.
+
+#. Can I use the same client package to run multiple instances for the same client?
+
+    Yes, you can start multiple instances of FL clients from the same client packages. Each FL client will be identified
+    by its unique instance names, for example: "flclient1", "flclient1_1", "flclient1_2", etc. The instance name must be
+    used for issuing specific commands to that client from the admin tool.
+
+#. What happens if a federated learning client crashes?
+
+    Federated learning clients will send a heartbeat call to the FL server once every minute. If an FL client crashes and
+    the FL server does not get a heartbeat from that client for 10 minutes (can be set with "heart_beat_timeout" in the
+    server's config json), the FL server will remove that client from the training client list.
+
+#. Can FL clients join or quit in the middle of federated learning training?
+
+    Yes, an FL client can join or quit in the middle of the FL training at any time. The client will pick up the global
+    model at the current round of the server to participate in the FL training. When quitting, the FL server will
+    automatically remove the FL client after it quits and no heartbeat is received for the duration of the
+    "heart_beat_timeout" configured on the server. If using an admin tool, it is recommended to use the "abort" and
+    "shutdown" commands to gracefully stop the clients.
+
+#. What if the number of participating FL clients is below the minimum number of clients required?
+
+    When an FL client passes authentication, it can request the current round of the global model and starts the FL training right away.
+    There is no need to wait for other clients. Once the client finishes its own training, it will send the update to the server
+    for aggregation. However, if the server does not receive enough updates from other clients, the FL server will not start
+    the next round of FL training. The finished FL client will be waiting for the next round's model.
+
+#. What happens if more than the minimum numbers of FL clients submit an updated model?
+
+    The FL server begins model aggregation after accepting updates from the minimum number of FL clients required and
+    waiting for "wait_after_min_clients" configured on the server. The updates that are received after this will be
+    discarded. All the clients will get the next round of the global model to start the next round FL training.
+
+#. How does a client decide to quit federated learning training?
+
+    The FL client always asks the server for the current round of training. If the server is not ready, the FL client will wait.
+    The client will only stop if the server becomes unreachable. The FL client can also be killed with the admin tool
+    issuing a "shutdown" command or by ctrl-C on the client itself, although this is not recommended because the server
+    will wait for the duration of "heart_beat_timeout" before knowing that the client has stopped.
+
+************************
+Server related questions
+************************
+
+#. What happens if the FL server crashes?
+
+    There are two scenarios for the FL server crashing during the FL training. If the server crashes when the FL client is trying to
+    connect to the server for model exchange, the FL client will continue to attempt connecting to the server for up to 30 seconds.
+    If the server is still down after that, FL client will shut itself down. If the server crashes during the FL client model training,
+    as long as the server restarts before the FL client attempts model updating, it will have no impact to the FL clients.
+
+    When restarting the FL server, you can find the previous training round number from the previous log. Then you can choose to
+    train from scratch or continuously using previous training model.
+
+#. Does the federated learning server need a GPU?
+
+    No, there is no need to have GPU on the server side for the FL server to deploy. However, certain handlers may require
+    GPUs. To disable GPUs on the server, include the following in the shell script that runs the server::
+
+        export CUDA_VISIBLE_DEVICES=
+
+#. What port do I need to open from the firewall on the FL server network?
+
+    Depending on the configuration of :ref:`project.yaml <project_yml>` which controls which port the gRPC is deployed to,
+    the FL server network needs to open that port for outside clients to reach the FL server.
+
+#. What if the federated learning server is behind a load balancer?
+
+    Currently, federated learning does not support load balancing between multiple FL servers.
+
+***************************************
+Overall training flow related questions
+***************************************
+
+#. How does the federated learning server decide when to stop FL?
+
+    The FL server always runs from the "start_round" to "num_rounds". The FL server will stop the training when the
+    current round meets "num_rounds".
+
+#. Can I run the FL server on AWS while running the FL client within my institution?
+
+    Yes, use the AWS instance name as the server cn in project.yml file. (e.g.: ec2-3-99-123-456.compute-1.amazonaws.com)
+
+#. How can I deploy different applications for different clients?
+
+    You can edit the application folder for each individual client on your desktop, then upload and deploy to each individual client
+    with the admin tool. Each client can run with its own application configuration.
+
+#. Can I use the same "run_number" as previously used?
+
+    Yes, you can re-use the same "run_number" as previously used. The "run_number" serves as an FL training workspace. The
+    FL training logs, such as tensorboard, training stats, etc, are stored within the same "run_number" workspace.
+
+#. What should I do if the admin notices one client's training is behaving erroneously or unexpectedly?
+
+    The admin can issue a command to abort the FL client training: ``abort client client_name``. If the command is issued
+    without the client_name, then the command will be sent to all the clients. Because of the nature of model training, it
+    may take a little time for the FL client to completely stop. Use the "check_status client client_name" command to see
+    if the client status is "stopped".
+
+#. Why do the admin commands to the clients have a long delay before getting a response?
+
+    The admin commands to the clients pass through the server. If for some reason the command is delayed by the network, or
+    if the client command takes a long time to process, the admin console will experience a delay for the response. The
+    default timeout is 10 seconds. You can use the “set_timeout” command to adjust the command timeout. If this timeout
+    value is set too low, the admin command may not reach the client to execute the command.
+
+#. Once the FL training has started, can I use the same server / client set up to train different models?
+
+    Yes, you can upload different applications to the server and clients to train different models. Make sure to use the
+    "run_number" to keep your trained models in different run spaces without confusing the models. The FL system only
+    completes when the admin issues the “shutdown” command or ctrl-C is used to end the process.
+
+#. Why if my custom components not updating between runs?
+
+    If you want to change the code you have already loaded in a custom component, it is recommended that you add a
+    version number or change the class name slightly. Python does not load new code definitions with the same class name
+    and by default Python does not allow the loaded modules to be removed. With a version or altered name, Python will
+    be able to treat the code as new and load it from the sys.path.
+
+#. Why do commands sometimes fail?
+
+    Sometimes if you are trying to check status of the client and the server is already busy transferring the model and
+    does not have extra bandwidth for the command, the command may time out. In that case, please wait and try again.
+
+*********************
+Cross Site Validation
+*********************
+
+#. I don't want to share my local model. Can I opt out of cross site validation?
+
+    Cross site validation is opt-out be default. To opt in, you must set "cross_site_validate" to true in config_fed_client.json.
+
+#. Cross site validation has finished. How can I see the results? 
+
+    Use the admin commands "validate all" or "validate <client_1> <client2>" to retrieve the results.
+
+#. Cross site validation ran but my results are empty. Why?
+
+    If some client is not participating in cross_site_validation OR an error occurs during validation, you will see empty results 
+    for that section. Please use the logs to retrieve the error.
+
+#. My client is stuck in endless loop of asking for models, then waiting and repeat. What do I do?
+
+    In some cases, cross site validation may get stuck. This is because the server sometimes doesn't know when (or if ever) a model 
+    will become available. In these cases, please use the "abort <client>" to stop cross site validation manually.
+
+#. I called "abort client" during training and it started cross site validation. Why?
+
+    This is the intended behavior. If a client is aborted, it will transition to cross site validation phase (if participating). To
+    completely abort, call "abort client" command again.
+
+************
+Known issues
+************
+
+#. If server dies and then is restarted, intentionally or unintentionally, all clients will have to be restarted.
+#. Running out of memory can happen at any time, especially if the server and clients are running on same machine.
+   This can cause the server to die unexpectedly.
+#. Putting applications in the transfer folders without using the upload_app command or forgetting to delete the models
+   folder inside, a mysterious error may occur when running the deploy_app command because the application folder is too
+   large to be uploaded and that causes timeout.
+#. Please don't start a new training run or start a new app before the previous application is fully stopped. Or users
+   can do ``abort client`` and ``abort server`` before ``start_app`` for the new run.
diff --git a/docs/favicon.ico b/docs/favicon.ico
new file mode 100644
index 0000000000..424df87200
Binary files /dev/null and b/docs/favicon.ico differ
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 0000000000..9d6e981605
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,32 @@
+############
+NVIDIA FLARE
+############
+
+NVIDIA FLARE (NVIDIA Federated Learning Application Runtime Environment) is a domain-agnostic, open-source, extensible
+SDK that allows researchers and data scientists to adapt existing ML/DL workflows (PyTorch, RAPIDS, Nemo, TensorFlow) to
+a federated paradigm; and enables platform developers to build a secure, privacy preserving offering for a distributed
+multi-party collaboration.
+
+Federated learning allows for multiple clients each with their own data to collaborate together without
+having to share their actual data. This allows for different parties located any place in the world to use their local
+secure protected data to perform tasks coordinated by an FL server set up in the secure cloud to achieve better learning
+results.
+
+NVIDIA FLARE is built on a componentized architecture, which allows researchers to customize workflows to their
+liking and experiment with different ideas quickly.
+
+Collaborative computing is not limited to deep learning, and with NVIDIA FLARE 2.0, you can develop any FL workflows
+with the newly provided `Controller API <programming_guide/controllers>`_. A full reference implementation of the
+`previous default workflow <programming_guide/controllers/scatter_and_gather_workflow>`_ is included along with the
+source code.
+
+.. toctree::
+   :maxdepth: 1
+
+   installation
+   quickstart
+   user_guide
+   programming_guide
+   faq
+   API <apidocs/modules>
+   appendix
diff --git a/doc/source/hello_installation.rst b/docs/installation.rst
similarity index 61%
rename from doc/source/hello_installation.rst
rename to docs/installation.rst
index f0f0fbc8e1..983c2678fa 100644
--- a/doc/source/hello_installation.rst
+++ b/docs/installation.rst
@@ -1,25 +1,23 @@
-Quickstart (Installation)
-==========================
+Installation
+=============
 
 Python Version
 --------------
 
-NVFlare requires `Python 3.8 <https://docs.python.org/3.8/>`_ or above.
+NVIDIA FLARE requires `Python 3.8.10 <https://www.python.org/downloads/release/python-3810/>`_.
 
+Install NVIDIA FLARE in virtual environments
+--------------------------------------------
 
-Install NVFlare in virtual environments
----------------------------------------
-
-It is highly recommended to install NVFlare in a virtual environment.
+It is highly recommended to install NVIDIA FLARE in a virtual environment.
 This guide briefly describes how to create a virtual environment with venv.
 
-
-Virtual Environments and Packages 
+Virtual Environments and Packages
 .................................
 
 Python's official document explains the main idea about virtual environments.
 The module used to create and manage virtual environments is called `venv <https://docs.python.org/3.8/library/venv.html#module-venv>`_.
-You can find more information there.  We only describe a few necessary steps for a virtual environment for NVFlare.
+You can find more information there.  We only describe a few necessary steps for a virtual environment for NVIDIA FLARE.
 
 
 Depending on your OS and the Python distribution, you may need to install the Python's venv package separately.  For example, in ubuntu
@@ -57,26 +55,29 @@ You may find that the pip and setuptools versions in the venv need updating:
   (nvflare-env) $ python3 -m pip install -U setuptools
 
 
-
 Install Stable Release
 ----------------------
 
-    
-Stable releases are available on `PyPI <https://pypi.org/>`_::
-
-    $ pip install nvflare
+Stable releases are available on `NVIDIA FLARE PyPI <https://pypi.org/project/nvflare>`_::
 
-.. warning::
+  $ python3 -m pip install nvflare
 
-    The above step (stable release) is not working.
-    
 
-Install Test Release
---------------------
+Install Nightly Release
+-----------------------
 
-From time to time, NVFlare is released to `Test PyPI <https://test.pypi.org/project/nvflare/>`_.
+NVIDIA FLARE also has a nightly build at `NVFlare nightly build <https://pypi.org/project/nvflare-nightly/>`_.
 You can install the package with the following command::
 
-  $ python3 -m pip install --extra-index-url https://test.pypi.org/simple nvflare
+  $ python3 -m pip install nvflare-nightly
 
 
+Clone Repository and Examples
+-----------------------------
+
+The next sections in the :ref:`quickstart` will guide you through the examples included in the repository. To clone the
+repo and get the source code:
+
+.. code-block:: shell
+
+  $ git clone https://github.com/nvidia/nvflare.git
diff --git a/docs/programming_guide.rst b/docs/programming_guide.rst
new file mode 100644
index 0000000000..540250022c
--- /dev/null
+++ b/docs/programming_guide.rst
@@ -0,0 +1,30 @@
+#####################################################
+Programming Guide - Developing Apps with NVIDIA FLARE
+#####################################################
+
+This guide introduces key objects and components you should know to implement your own components.
+
+NVIDIA FLARE is designed with a componentized architecture which allows users to bring customized components into the
+framework.
+
+Different components can be built on top of the APIs(:mod:`nvflare.apis`) in NVIDIA FLARE core, and you can now
+implement your own custom workflows. The concepts of aggregator, learnable, persistors, and shareable_generator which
+were fixed in the workflow have been decoupled from the core of NVIDIA FLARE and moved to :mod:`nvflare.app_common`.
+This is also the package containing the reference implementation of the :ref:`Scatter and Gather Workflow`, and all of
+this can be used in your own workflow.
+
+If you want to use your components in an FL application, you will need to change the config in the application folder.
+Please refer to :ref:`user_guide/application:NVIDIA FLARE Application` for more details.
+
+.. toctree::
+   :maxdepth: 1
+
+   programming_guide/fl_context
+   programming_guide/fl_component
+   programming_guide/controllers
+   programming_guide/filters
+   programming_guide/executor
+   programming_guide/event_system
+   programming_guide/shareable
+   programming_guide/data_exchange_object
+   programming_guide/best_practices
diff --git a/docs/programming_guide/best_practices.rst b/docs/programming_guide/best_practices.rst
new file mode 100644
index 0000000000..5b1b777b04
--- /dev/null
+++ b/docs/programming_guide/best_practices.rst
@@ -0,0 +1,300 @@
+NVIDIA FLARE Programming Best Practices
+=======================================
+.. currentmodule:: nvflare.apis.fl_component.FLComponent
+Define your logic in a subclass of FLComponent
+----------------------------------------------
+NVIDIA FLARE has a componentized architecture - business logic is implemented by components that interact with each
+other to get the job done. Most of your code, if not all, should be implemented as subclasses of the
+:class:`FLComponent<nvflare.apis.fl_component.FLComponent>` class.
+
+There are many benefits of being a FLComponent:
+
+    - All such components are automatically event handlers
+    - Convenience methods for logging, and firing events
+    - Error log streaming to server for centralized error management (if enabled)
+    - Well-defined component lifecycle
+
+Simple use of FLContext
+-----------------------
+:class:`FLContext<nvflare.apis.fl_context.FLContext>` caters relevant data to your component's processing methods. All
+methods of FLComponent have a fl_ctx input arg.
+
+The fl_ctx object typically contains the following information:
+
+    - The overall system settings (identity name, workspace, run-time args, run abort signal, etc.): All FLContext
+      objects have these, and there are convenience methods for them.
+    - Event data: When an event is fired, the event poster usually places some event specific data into the FLContext
+      before firing the event. Event handlers can access such data in their handle_event() methods.
+
+You could also use the fl_ctx to pass around information between your own functions, but make sure all these function
+calls happen within the same processing stack of your own component.
+
+Do not assume all components share the same fl_ctx and use fl_ctx to pass information between different components. If
+you want to reliably pass a piece of information to others, use event instead: you fire an event type to announce the
+information, and others receive the information by handling your event.
+
+Agreement on Data Format and Content
+------------------------------------
+FL is data-based collaboration among different parties (components running on server and clients). Having
+an accurate understanding of the shared data is a basic requirement. For example, the model weights generated by the
+trainer on the client must be consumable by the aggregator running on the server.
+
+NVIDIA FLARE encourages data content to be self-sufficient in that it provides not only the content (e.g. values of
+model weights), but also complete contextual/meta information about the content (e.g. whether value is model weights or
+weight diffs, whether the value has been processed by any algorithm, and any processed properties).
+
+To enable easy adoption of this strategy, NVIDIA FLARE provides a simple DXO API (Data eXchange Object) that you can use
+to describe the data you want to communicate between server and clients.
+
+Use Filters to accommodate data differences
+-------------------------------------------
+When two parties get into disagreements on the data format, you may modify one of them to accommodate the other. This
+is doable but may not be the best solution, since it makes code hard to reuse.
+
+A better solution is to use filters. As long as both parties agree on the data content (not format), you can write a
+filter to convert the data from one format to another, so both parties can stay intact.
+
+Filters can do a lot more than just converting data formats. You can apply any type of massaging to the data for the
+purpose of security. In fact, NVIDIA FLARE provided privacy protection techniques are all implemented as filters.
+
+Be aware of the running environment
+-----------------------------------
+NVIDIA FLARE is a messaging system running in a multithreaded environment. Care must be taken when programming
+with NVIDIA FLARE to ensure smooth operation of the system.
+
+    - Methods of a component may be running in different threads. Controller callbacks are running in threads that
+      receive messages on a connection with the client.
+    - The control_flow() method of the controller runs in a dedicated thread throughout the RUN.
+    - In your own component, you can create more separate threads for your own design logic.
+    - Events could be fired from any thread, and the :meth:`handle_event()<handle_event>` method runs within the same
+      thread from which the event is fired.
+    - All components are running within a RUN, and components are created and released at the beginning and end of the RUN.
+
+Your component may have many methods. Keep in mind that these methods may be running in different threads at the same
+time!
+
+Manage your component's life cycle gracefully
+---------------------------------------------
+During the life cycle of your component, things may happen that could interrupt the normal processing of the component.
+For example, the admin user or the workflow controller may ask to terminate the current RUN; or to terminate the
+execution of the current task on the client.
+
+Since many components and threads could be involved in the RUN, it is important to ensure graceful exits of the
+components and threads, for the smooth operation of the FL system.
+
+To manage these gracefully, follow the following rule-of-thumbs.
+
+Respect the abort signal
+------------------------
+An abort-signal is a simple Python object that can be triggered to indicate that either the RUN or Task should be ended.
+There are two kinds of abort signals:
+
+    - **RUN abort signal**. When triggered, it indicates that the current RUN is to be aborted. If you have long-running
+      logic, you should check this signal frequently and exit when it is triggered.
+    - **Task Execution abort signal**. This is the abort_signal passed to the execute() method of an executor (on the
+      client side). If you write executors, make sure to check this signal frequently in the execute() method and return
+      from the method when the signal is triggered.
+
+.. note::
+
+    You can always get the RUN abort_signal from an FL context: ``fl_ctx.get_abort_signal()``. You can also keep it with
+    your component object when starting your component (i.e. when handling the EventType.START_RUN event).
+
+.. note:: You cannot get the Task Abort Signal from an FL Context!
+
+The triggering of abort signals is the responsibility of the framework. NEVER try to trigger them in your component!
+
+If your component runs into a condition that definitely requires drastic actions, call self.system_panic() to abort the
+whole RUN, or self.task_panic() to end the task execution (in the Executor on client).
+
+Handle EventType.ABORT_TASK event
+---------------------------------
+What if you use a 3rd-party solution to run a long-running execution in your executor, and the 3rd-party solution does
+not take the abort_signal?
+
+If the 3rd-party solution does not provide any way to interrupt/end its execution prematurely, then you are out of
+luck - the task cannot be aborted, and the system will just have to wait until the end of the execution. Though this
+will not cause any logical errors, it could cause the overall slowdown of the FL system, because the client won't be
+able to fetch the next task until the current task is done.
+
+If the 3rd-party solution provides a method you can call from a different thread to end its execution, then you are
+lucky. In addition to triggering the task abort signal, NVIDIA FLARE also fires the special event EventType.ABORT_TASK!
+You just need to handle this event in your executor's handle_event() method and call the 3rd-party provided method to
+stop the execution.
+
+Avoid recursive events if possible
+----------------------------------
+You can fire another event in your event handling logic. But try to avoid this if possible so as to avoid potential
+dead-loops. If you have to do it, be very careful that it won't cause cyclic event firing.
+
+.. note::
+
+    The system has a built-in protection against cyclic event firing - if the firing goes beyond certain predefined
+    depth, the firing will be stopped.
+
+.. tip::
+
+    Firing an event from an event handler could be useful in some cases. For example, NVIDIA FLARE's event system does
+    not enforce any order when invoking handle_event() methods of event handlers, hence you cannot assume some handlers
+    are invoked before some other handlers. If you must ensure handler A is invoked before B for an event type E, you
+    can do it yourself: in A's handle_event() method, you fire a new event E2 when handling E, and B only handles E2.
+
+If you develop workflow controllers
+-----------------------------------
+Use a new Task object for each task
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Do not reuse :class:`Task<nvflare.apis.controller_spec.Task>` objects -- always create a new Task object when scheduling
+tasks instead. Try to avoid using the same "task name" for different functions.
+
+For maximum flexibility, make your task names configurable so you only need to define them in the JSON config file at
+the time of app creation.
+
+Set reasonable timeouts for your tasks
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+In the ``control_flow()`` method of your controller, you will create tasks. Don't forget to set a reasonable timeout value
+(number of secs for the task to complete). The default value is 0, which means never timeout. If you don't specify a
+non-zero timeout, under certain circumstances, your task may just keep on waiting until clients start the RUN.
+
+Callbacks should be short-lived
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Since callbacks are running within a connection thread, they must return very quickly to release the underlying
+connection; otherwise the whole system could become frozen when all connections are used.
+
+Task props usage
+^^^^^^^^^^^^^^^^
+.. currentmodule:: nvflare.apis.controller_spec.Task
+Once a task is created, do not modify the task.props dict directly (e.g. task.props = {"key": "value"}) since this dict
+is carefully managed by the framework.
+
+Instead, always use :meth:`set_prop()<set_prop>`::
+
+    task.set_prop("key", "value")
+
+Task completion_status
+^^^^^^^^^^^^^^^^^^^^^^
+If the task.completion_status is not None, this task will be treated as finished or errored-out. It will not be sent
+to any clients.
+
+Easy Config of your components
+------------------------------
+To make it easy to configure your components in JSON, use only primitive data types for init args: number, str, list,
+dict. Never use Python objects.
+
+Nested Components?
+------------------
+Sometimes your component is composed of other parts (Python objects). How do you get your component created with all the parts instantiated? 
+
+Use component ID!
+
+    - Parts should be configured in the "components" section of the config. You need to specify a unique ID for each part.
+    - In the configuration of the parent component, specify the ID of the part as an init arg in the JSON configuration file.
+    - When handling the EventType.START_RUN event, get the part object from the engine and keep it in the parent component::
+
+        engine = fl_ctx.get_engine()
+        self.my_part = engine.get_component(self.my_part_id)
+
+Manage your resources properly
+------------------------------
+The life span of your components is that of a RUN.
+
+Your components may use resources such as memory, threads, and even processes. Make sure to manage them properly,
+following this pattern:
+
+    - In __init__(), create your resources but do not start them.
+    - In handling EventType.START_RUN, start the resources (e.g. open files, start threads, etc.).
+    - In handling EventType.END_RUN, terminate the resources (e.g. close files, join threads and/or processes). Make
+      sure to test the resources before terminating them (e.g. if self.thread.is_active()).
+
+Do not terminate your resources before EventType.END_RUN.
+
+Promote Decoupled Component Interactions
+----------------------------------------
+.. currentmodule:: nvflare.apis.fl_component.FLComponent
+Throughout the FL execution, many pieces of data are generated. What if you want to do something about that data? You
+may have the urge to modify the code that generated the data and insert your logic there to manipulate the data. Don't!
+Use event handling instead.
+
+NVIDIA FLARE's event system is like a pub-sub mechanism. The publisher fires an event, and subscribers handle the event.
+The mechanism even works across network boundaries - a component on the server can post an event and components on
+the server and components on clients can handle the event; and vice versa.
+
+Fire an event for the following cases:
+
+    - Your component generates a piece of data that could be useful for other components. You should name the event type to be like "datatype_available" where datatype specifies the type of the data.
+    - Your component's processing flow has come to a point that could be interesting to others. You should name event types like: "before_somepoint" and "after_somepoint", where somepoint specifies the point.
+
+To fire an event, call :meth:`self.fire_event()<fire_event>` or :meth:`self.fire_fed_event()<fire_fed_event>`.
+
+What events should you fire? Well, it's up to your imagination whether the event could be useful to others.
+
+Event Handling
+--------------
+If you are interested in the data or timing of an event type that is fired by another component, you can handle that
+event type in the :meth:`handle_event()<handle_event>` method of your component.
+
+Event data is contained in the FLContext object that is passed to the :meth:`handle_event()<handle_event>` method. Unless absolutely
+necessary, you should treat data in the FLContext as read-only.
+
+Naming of Event Types, FLContext props, Shareable keys, and Headers
+-------------------------------------------------------------------
+Name the names with only letters, digits and underscores. Do not prefix the key names with underscores - such names are
+for the internal use of NVIDIA FLARE framework.
+
+The name of the fed event type should be prefixed with "fed.".
+
+Use Shareable Properly
+----------------------
+Shareable is just a dictionary. However observe these rules of thumb:
+
+    - The headers element (ReservedHeaderKey.HEADERS) is reserved. Never overwrite it!
+    - You can add additional props into the headers element, but never use reserved header names.
+
+Do not share FLContext objects across different threads
+-------------------------------------------------------
+To avoid potential race conditions that could compromise data integrity of FLContext objects, avoid sharing the same
+FLContext across in multiple threads. The framework guarantees to create a new FLContext object for each messaging
+thread. If your component creates its own thread, that thread can create a FLContext object with engine.new_context().
+
+.. tip:: The Engine object can be shared across threads.
+
+Handle exceptions
+-----------------
+Things can go wrong unexpectedly, even if there are no logical errors in your code. You should place your logic in the
+``try`` block and handle exceptions gracefully. If not, the framework will catch the exception and may take actions that
+may not be what you wanted (e.g. stop the whole RUN).
+
+Panic or Not
+------------
+To create solid components that work well in real-world apps, you are encouraged to do "defensive programming" - trust
+nothing and test everything, and cover all kinds of edge conditions. In doing so, you may come to face some extreme
+situations: what to do when a combination of conditions makes the whole thing just unworkable? For example, none of the
+Clients sends you meaningful training results. Should you just extend the number of rounds and see whether the clients
+will get better?
+
+If you decide that the situation is truly unworkable, you can simply end the whole RUN by calling :meth:`self.system_panic()<system_panic>`.
+Similarly, if you run into a bad situation while executing a task (on client), you can end the task by calling
+:meth:`self.task_panic()<task_panic>`.
+
+The panic methods are effective regardless of component types and which thread it is called from.
+
+Logging
+-------
+.. currentmodule:: nvflare.apis.fl_component.FLComponent
+Instead of directly using self.logger to log messages, always use the methods :meth:`self.log_debug<log_debug>`,
+:meth:`self.log_info<log_info>`, :meth:`self.log_warning<log_warning>`, :meth:`self.log_error<log_error>` instead.
+These methods provide many benefits:
+
+    - All log messages are prefixed with some contextual information (e.g. run number, peer name, peer run number, etc.) that can help you understand the log messages more easily.
+    - Some log messages may be integrated with other system features. For example, error messages may be reported via admin commands, and/or streamed over to a centralized location.
+
+Be Sensitive to Data Privacy
+----------------------------
+Data is exchanged between the Server and Clients in following situations:
+
+    - Task execution
+    - Fed events
+    - Aux channel communication
+    - Centralized error reporting
+
+Be very careful about the data sent to peers! Make sure it never contains privacy sensitive information. This
+also applies to error messages, since they may be sent to a centralized location. 
diff --git a/docs/programming_guide/controllers.rst b/docs/programming_guide/controllers.rst
new file mode 100644
index 0000000000..cabcb0c676
--- /dev/null
+++ b/docs/programming_guide/controllers.rst
@@ -0,0 +1,76 @@
+Controllers and Controller API
+==============================
+
+Controller/Worker Interactions
+------------------------------
+
+NVIDIA FLARE 2.0's collaborative computing is achieved through the Controller/Worker interactions.
+
+.. image:: ../resources/Controller_worker.png
+    :height: 300px
+
+The Controller is a python object that controls or coordinates the Workers to get a job done. The controller is run on
+the FL server (highlighted on the right).
+
+.. image:: ../resources/Controller.png
+    :height: 300px
+
+A Worker is capable of performing tasks. Workers run on FL clients.
+
+In its control logic, the Controller assigns tasks to Workers and processes task results from the Workers.
+
+Workers keep asking for the next task to do, executes the task, and submits results to the Controller, until instructed
+to exit by the Controller (a special END_RUN task).
+
+The following diagram depicts how the Controller and Worker interact.
+
+Controller API
+--------------
+
+The :mod:`Controller API<nvflare.apis.controller_spec>` provides methods for assigning tasks to the Workers (FL clients)
+in different ways:
+
+   - Broadcast a task to multiple clients
+   - Send a task to a single client
+   - Arrange a task to be done by multiple clients in turns
+
+See the included :class:`Controller<nvflare.apis.impl.controller.Controller>` implementation and full reference
+implementations of the following controller workflows: :ref:`controllers/scatter_and_gather_workflow`, :ref:`controllers/cross_site_model_evaluation`, and
+:ref:`controllers/cyclic_workflow`.
+
+.. include:: controllers/scatter_and_gather_workflow.rst
+
+.. include:: controllers/cross_site_model_evaluation.rst
+
+.. include:: controllers/cyclic_workflow.rst
+
+You can study the source code and use it as a starting point to write your own controller workflows.
+
+Task Lifecycle
+--------------
+
+The central concept of the Controller API is :class:`Task<nvflare.apis.controller_spec.Task>`.
+
+A :class:`Task<nvflare.apis.controller_spec.Task>` is a piece of work that is assigned by the Controller to client workers. Depending on how the task is assigned (broadcast, send, or relay), the task will be performed by one or more clients.
+
+The Controller's Task Manager manages the task's lifecycle:
+
+    - First, the programmer creates the task, specifying the name and the data of the task.
+    - Then, the programmer calls one of the task methods (e.g. broadcast, send, relay, etc.). All these methods do is simply adding the task to the Task Queue. Now the task is waiting for clients to come to retrieve it. Note that there could be multiple tasks in the queue.
+    - When a client comes to get the next task, the Task Manager decides which task in the queue should be assigned to the client. The general rule is that the tasks will be examined one by one following their orders in the queue. If the client is a candidate of the task and the task has not been performed by the client, AND the task-specific rule allows the client to be assigned, then the task is assigned to the client, and a new ClientTask record is created and added to the task's client_tasks list. If the before_task_sent callback (CB) is provided, it is called before sending the task to the client.
+    - If no task is found for the client, the Task Manager tells the client to try again later.
+    - When the client finishes its assigned task and comes back to submit its result, the client_task is found for this client, and then the result_received CB (if provided) is called. The result is recorded into the client_task record, and the client_task is marked as "result received".
+    - Eventually the task is completed when one of the following conditions is met:
+        - The task itself is timed out (if the task timeout is specified)
+        - All assigned tasks received results from clients
+        - Task specific exit rule is met (e.g. for broadcast, the minimal-responses are received and waited for enough time after that)
+        - The task is cancelled explicitly
+        - Fatal error occurred (task data filtering error) and the task is cancelled by the system
+    - Once the task is completed, it's completion_status is set based on the condition the task is completed, and the task is removed from the task queue. If the task_done CB is provided, it is called. This is the end of the task's lifecycle.
+
+.. note::
+
+    In NVIDIA FLARE 2.0, the underlying communication is by gRPC: the client always initiates communication by sending
+    a request to the server and a receiving response. When we say "server sends task to the client", it is only
+    conceptual. With gRPC, the client sends the "ask for next task" request to the server, and the server responds with
+    the task data.
diff --git a/docs/programming_guide/controllers/cross_site_model_evaluation.rst b/docs/programming_guide/controllers/cross_site_model_evaluation.rst
new file mode 100644
index 0000000000..7133f3681c
--- /dev/null
+++ b/docs/programming_guide/controllers/cross_site_model_evaluation.rst
@@ -0,0 +1,5 @@
+Cross Site Model Evaluation / Federated Evaluation
+==================================================
+The cross site model evaluation workflow uses the data from clients to run evaluation with the models of other clients.
+
+:class:`nvflare.app_common.workflows.cross_site_model_eval.CrossSiteModelEval`
diff --git a/docs/programming_guide/controllers/cyclic_workflow.rst b/docs/programming_guide/controllers/cyclic_workflow.rst
new file mode 100644
index 0000000000..6c6f32a9dc
--- /dev/null
+++ b/docs/programming_guide/controllers/cyclic_workflow.rst
@@ -0,0 +1,5 @@
+Cyclic Workflow
+===============
+Cyclic workflow is new for NVIDIA FLARE 2.0.
+
+:class:`nvflare.app_common.workflows.cyclic_ctl.CyclicController`
diff --git a/docs/programming_guide/controllers/scatter_and_gather_workflow.rst b/docs/programming_guide/controllers/scatter_and_gather_workflow.rst
new file mode 100644
index 0000000000..ce956f3d03
--- /dev/null
+++ b/docs/programming_guide/controllers/scatter_and_gather_workflow.rst
@@ -0,0 +1,42 @@
+Scatter and Gather Workflow
+============================
+The Federated scatter and gather workflow is an included reference implementation of the default workflow of previous versions
+of NVIDIA FLARE with a Server aggregating results from Clients that have produced Shareable results from their Trainer.
+
+At the core, the control_flow of :class:`nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather` is a for loop:
+
+.. image:: ../../resources/fed_avg_round.png
+    :height: 400px
+
+Trainer
+-------
+You need to implement the ``train()`` method for your :class:`Trainer<nvflare.app_common.executors.trainer.Trainer>`.
+
+FL client gets the global ``Shareable`` from the FL server for each round and then the ``train()`` method will be called.
+
+The ``train()`` method needs to get the required information from the global ``Shareable``,
+use that in its training process, then returning the local training result as a ``Shareable``.
+
+You will need to configure your own ``Trainer`` in config_fed_client.json.
+Example FL configurations can be found in :ref:`user_guide/application:NVIDIA FLARE Application`.
+
+Learnable
+---------
+:class:`Learnable<nvflare.app_common.abstract.learnable.Learnable>` is the result of an FL application.
+For example, in the deep learning scenario, it can be the model weights.
+In the AutoML case, it can be the network architecture.
+
+A :class:`LearnablePersistor<nvflare.app_common.abstract.learnable_persistor.LearnablePersistor>` defines how to load
+and save a ``Learnable``. ``Learnable`` is a subset of the model file (which can contain other data like LR schedule)
+which is to be learned, like the model weights.
+
+Aggregator
+----------
+:class:`Aggregators<nvflare.app_common.abstract.aggregator.Aggregator>` define the aggregation algorithm to aggregate the ``Shareable``.
+For example, a simple aggregator would be just average all the ``Shareable`` of the same round.
+
+Below is the signature for an aggregator.
+
+.. literalinclude:: ../../../nvflare/apis/aggregator.py
+    :language: python
+    :lines: 22-
diff --git a/docs/programming_guide/data_exchange_object.rst b/docs/programming_guide/data_exchange_object.rst
new file mode 100644
index 0000000000..86aba8065f
--- /dev/null
+++ b/docs/programming_guide/data_exchange_object.rst
@@ -0,0 +1,18 @@
+Data Exchange Object (DXO)
+==========================
+.. currentmodule:: nvflare.apis.dxo.DXO
+
+The Data Exchange Format (:class:`nvflare.apis.dxo.DXO`) in NVIDIA FLARE standardizes the data passed between the communicating parties.
+
+.. literalinclude:: ../../nvflare/apis/dxo.py
+    :language: python
+    :lines: 29-40
+
+``data_kind`` keeps track of the kind of data for example "WEIGHTS" or "WEIGHT_DIFF".
+
+``meta`` is a dict that can contain additional properties.
+
+The method :meth:`to_shareable()<to_shareable>` produces a :ref:`shareable`, and a DXO can be retrieved from a
+:ref:`shareable` with :meth:`nvflare.apis.dxo.from_shareable`.
+
+It is recommended to use DXO to maintain consistency in managing the data throughout the FL system.
diff --git a/docs/programming_guide/event_system.rst b/docs/programming_guide/event_system.rst
new file mode 100644
index 0000000000..b4098f5f11
--- /dev/null
+++ b/docs/programming_guide/event_system.rst
@@ -0,0 +1,128 @@
+NVIDIA FLARE Event Mechanism
+============================
+NVIDIA FLARE comes with a powerful event mechanism that allows dynamic notifications sent to all objects that are of a
+subclass of :class:`FLComponent<nvflare.apis.fl_component.FLComponent>`.
+
+All component types in NVIDIA FLARE (e.g. Filter, Executor, Responder, Controller, Widget, etc.) are subclasses of
+``FLComponent``, hence they are all event handlers. You can create additional subclasses, and their objects will
+automatically become event handlers too.
+
+Events
+------
+Events represent some important moments during the execution of the system logic. Following are typical examples of
+such moments:
+
+    - Before some action happens (e.g. before aggregation)
+    - After some action happens (e.g. after aggregation)
+    - When some important data becomes available (e.g. best model updated)
+
+You can freely invent your own event types and fire events in your processing logic.
+
+Event Type
+^^^^^^^^^^
+An event type is simply a string. Since anyone can invent a new event type, event naming conventions should be defined
+to avoid possible name conflicts.
+
+Event Data
+^^^^^^^^^^
+Event data can be anything. Most (but not all) event types have data. Event data is placed into the FLContext object as
+a property under a well-defined name. Event handlers that listen to the event type get the event data with FLContext's
+get_prop() method.
+
+Not all event types have data - some events are merely used for the timing of the events.
+
+Firing Events
+^^^^^^^^^^^^^
+To fire an event in your processing logic:
+    - First, decide when to fire the event in your processing logic
+    - Second, if the event has some data, add it to the fl_ctx. Note that in most cases you should already have a
+      fl_ctx. If not, you can create a new one with engine.new_context().
+    - Finally, fire the event.
+
+The following is a typical code pattern::
+
+    fl_ctx.set_prop(key="yourEventDataKey",
+                      data=yourEventData,
+                      private=True,
+                      sticky=False)
+    engine = fl_ctx.get_engine()
+    engine.fire_event(event_type="youEventTypeName",
+                        fl_ctx=fl_ctx)
+
+.. note::
+
+    Usually the event data should be private and non-sticky. But if you also want to keep the data around permanently
+    during the RUN, you can set it to sticky.
+
+.. note::
+
+    If your event could have multiple pieces of data, you can simply call fl_ctx.set_prop() multiple times - one for
+    each piece of data.
+
+If your event doesn't have any data, then you can skip the first step.
+
+Handling Events
+^^^^^^^^^^^^^^^
+When an event is fired, the handle_event() method of all event handling components are invoked to process the event.
+
+.. note::
+
+    The order in which the components are invoked is non-deterministic. Therefore you cannot rely on a component being
+    invoked before another. This behavior may change in the future if needed.
+
+The following is a typical code pattern for handling events you are interested in::
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+       if event_type == 'event1':
+           event_data = fl_ctx.get_prop('eventDataKeyName')
+           ...
+       elif event_type == 'event2':
+           # process
+
+Event handlers are called in sequence. The failure (exception) of one handler does not stop the invocation of subsequent handlers.
+
+Built-in Event Types
+--------------------
+NVIDIA FLARE's system-defined event types are specified in :class:`nvflare.apis.event_type.EventType`:
+
+todo: go over these
+
+.. csv-table::
+   :header: Event, Description
+
+    START_RUN, Start of the current "run"
+    END_RUN, End of the current "run"
+    CLIENT_REGISTER, When the client register to the server
+    CLIENT_QUIT, When the client quit from the server.
+    GET_PULL_REQUEST, .
+    GET_SUBMIT_RESULT, .
+    BEFORE_PULL_TASK, Before the .
+    AFTER_PULL_TASK, After the .
+    COLLECT_STATUS, .
+    BEFORE_PROCESS_SUBMISSION, Before the .
+    AFTER_PROCESS_SUBMISSION, After the .
+    BEFORE_TASK_DATA_FILTER, Before the .
+    AFTER_TASK_DATA_FILTER, After the .
+    BEFORE_TASK_RESULT_FILTER, Before the .
+    AFTER_TASK_RESULT_FILTER, After the .
+    BEFORE_TASK_EXECUTION, Before the .
+    AFTER_TASK_EXECUTION, After the .
+    BEFORE_SEND_TASK_RESULT, Before the .
+    AFTER_SEND_TASK_RESULT, After the .
+
+RUN Lifecycle Events
+--------------------
+The most important event types of all are START_RUN and END_RUN.
+
+In NVIDIA FLARE, a FL experiment is conducted in a RUN. During the course of a FL study, researchers usually need to conduct
+many RUNs to achieve expected results.
+
+START_RUN event occurs when a new RUN is about to start, usually triggered by the researcher via admin commands. If your
+component needs to be initialized, you must listen to this event type and get your component ready for work.
+
+The END_RUN event occurs when the RUN is about to end, usually triggered by completion of the workflow or the abort
+command from the researcher. If needed, you can use this event to gracefully finalize and/or clean up your component.
+
+If your component provides some services that other components can use, you can put your component as a private and
+sticky prop into the fl_ctx at the time of START_RUN under a uniquely defined prop name. Other components can later get
+your component with this name and invoke the services of your component.
diff --git a/docs/programming_guide/executor.rst b/docs/programming_guide/executor.rst
new file mode 100644
index 0000000000..56626300f5
--- /dev/null
+++ b/docs/programming_guide/executor.rst
@@ -0,0 +1,101 @@
+NVIDIA FLARE Executors
+======================
+
+.. image:: ../resources/Executor.png
+    :height: 300px
+
+An :class:`Executor<nvflare.apis.executor.Executor>` in NVIDIA FLARE is a type of FLComponent for FL clients that has an
+``execute`` method that produces a Shareable from an input Shareable. The ``execute`` method also takes a str for
+task_name, FLContext, and abort_signal.
+
+.. literalinclude:: ../../nvflare/apis/executor.py
+    :language: python
+    :lines: 24-
+
+Examples for Executors are :class:`Trainer<nvflare.app_common.executors.trainer.Trainer>` and :class:`Validator<nvflare.app_common.executors.validator.Validator>`.
+The source code for some example implementations can be found in the example apps. On clients, tasks can be configured
+for Executors in config_fed_client.json:
+
+.. code-block:: json
+
+    {
+      "format_version": 2,
+      "handlers": [],
+      "executors": [
+        {
+          "tasks": [
+            "train",
+            "submit_model"
+          ],
+          "executor": {
+            "path": "np_trainer.NPTrainer",
+            "args": {}
+          }
+        },
+        {
+          "tasks": [
+            "validate"
+          ],
+          "executor": {
+            "path": "np_validator.NPValidator"
+          }
+        }
+      ],
+      "task_result_filters": [],
+      "task_data_filters": [],
+      "components": []
+    }
+
+The above configuration is an example from hello_numpy. Each task can only be assigned to one Executor.
+
+Multi-Process Executor
+----------------------
+:class:`MultiProcessExecutor<nvflare.app_common.executors.multi_process_executor.MultiProcessExecutor>` is designed to
+easily allow the FL executor to support multi-processes execution. The behavior of the Executor remains the same
+including the firing and handling of FL events. MultiProcessExecutor allows researchers to focus on the training and
+execution logic instead of worrying about how to make use of multiple processes or deal with multi-GPU training.
+
+During the execution, any event fired from other components will be relayed from the MultiProcessExecutor to all the
+sub-worker processes. Any component which listens to the event in the sub-worker processes can handle the event
+accordingly. Also, any event fired by the FL component in the sub-worker processes will be relayed by the
+MultiProcessExecutor to all other components to handle.
+
+.. image:: ../resources/multi_process_executor.png
+    :height: 400px
+
+MultiProcessExecutor keeps the same FL Executor API signature. When turning the FL executor into
+MultiProcessExecutor, configure the task executor to use MultiProcessExecutor (currently PTMultiProcessExecutor is the
+only implemented MultiProcessExecutor), and configure the existing executor as the "executor_id", and the number of
+processes to use.
+
+.. code-block:: json
+
+    {
+      "executors": [
+        {
+          "tasks": [
+            "train"
+          ],
+          "executor": {
+            "path": "nvflare.app_common.pt.pt_multi_process_executor.PTMultiProcessExecutor",
+            "args": {
+              "executor_id": "trainer",
+              "num_of_processes": 2,
+              "components": [
+                {
+                  "id": "trainer",
+                  "path": "medl.apps.fed_learn.trainers.client_trainer.ClientTrainer",
+                  "args": {
+                    "local_epochs": 5,
+                    "steps_aggregation": 0,
+                    "model_reader_writer": {
+                      "name": "PTModelReaderWriter"
+                    }
+                  }
+                }
+              ]
+            }
+          }
+        }
+      ],
+    }
diff --git a/docs/programming_guide/filters.rst b/docs/programming_guide/filters.rst
new file mode 100644
index 0000000000..ba1ff20855
--- /dev/null
+++ b/docs/programming_guide/filters.rst
@@ -0,0 +1,26 @@
+NVIDIA FLARE Filters
+====================
+Filters in NVIDIA FLARE are a type of FLComponent that has a ``process`` method to transform the ``Shareable`` object between
+the communicating parties. A ``Filter`` can be used to provide additional processing to shareable data before sending or
+after receiving from the peer.
+
+The ``FLContext`` is available for the ``Filter`` to use.
+
+.. literalinclude:: ../../nvflare/apis/filter.py
+    :language: python
+    :lines: 22-
+
+In config_fed_server.json and config_fed_client.json (for details see :ref:`user_guide/application:NVIDIA FLARE Application`),
+task_result_filters and task_data_filters can be configured for processing data at the points
+highlighted in the image below:
+
+.. image:: ../resources/Filters.png
+    :height: 350px
+
+Filters can convert data formats and a lot more. You can apply any type of massaging to the data for the
+purpose of security. In fact, privacy and homomorphic encryption techniques are all implemented as filters:
+
+    - ExcludeVars to exclude variables from shareable (:mod:`nvflare.app_common.filters.exclude_vars`)
+    - PercentilePrivacy (:mod:`nvflare.app_common.filters.percentile_privacy`)
+    - SVTPrivacy (:mod:`nvflare.app_common.filters.svt_privacy`)
+    - Homomorphic encryption (:mod:`nvflare.app_common.homomorphic_encryption.he_model_encryptor.py` and :mod:`nvflare.app_common.homomorphic_encryption.he_model_decryptor`)
diff --git a/docs/programming_guide/fl_component.rst b/docs/programming_guide/fl_component.rst
new file mode 100644
index 0000000000..cb5ebc3235
--- /dev/null
+++ b/docs/programming_guide/fl_component.rst
@@ -0,0 +1,30 @@
+FLComponent
+===========
+.. currentmodule:: nvflare.apis.fl_component.FLComponent
+
+:class:`nvflare.apis.fl_component.FLComponent` is the base class of all the FL components. Executors, controllers, filters, aggregators, and their subtypes for
+example trainer are all FLComponents now.
+
+.. literalinclude:: ../nvflare/apis/fl_component.py
+    :language: python
+    :lines: 7-21
+
+Each ``FLComponent`` is automatically added as an event handler in the system when a new instance is created.
+You can implement the :meth:`handle_event<handle_event>` to plugin additional customized actions to the FL workflows.
+
+To fire events, :meth:`fire_event<fire_event>` can be used, and :meth:`fire_fed_event<fire_fed_event>` can be used to
+fire an event across participants.
+
+The logging methods :meth:`log_debug<log_debug>`, :meth:`log_info<log_info>`, :meth:`log_warning<log_warning>`,
+:meth:`log_error<log_error>`, and :meth:`log_exception<log_exception>` should be used to prefix log messages with
+contextual information and integrate with other system features.
+
+In extreme cases where the system encounters errors that prevent further operation, :meth:`task_panic<task_panic>` can
+be called to end the task, or :meth:`system_panic<system_panic>` can be called to end the run.
+
+Default data in the built-in FLComponents
+-----------------------------------------
+For the built-in FLComponents provided by NVIDIA FLARE, we assure the following data is set in the ``Shareable`` and ``FLContext``.
+
+You can also define the structure of ``Sharable`` objects that fits your needs and
+add your training associated data into ``FLContext``.
diff --git a/docs/programming_guide/fl_context.rst b/docs/programming_guide/fl_context.rst
new file mode 100644
index 0000000000..d480701b38
--- /dev/null
+++ b/docs/programming_guide/fl_context.rst
@@ -0,0 +1,317 @@
+FLContext
+=========
+.. currentmodule:: nvflare.apis.fl_context.FLContext
+One of the most important features of NVIDIA FLARE is :mod:`nvflare.apis.fl_context` to pass data between the FL
+components. ``FLContext`` is available to every method of all FLComponent types (Controller, Aggregator, Filter,
+Executor, Widget).
+
+Through the FL Context, the component developer can:
+    - Get services provided by the underlying infrastructure
+    - Share data with other components of the FL system, even including components in the peer endpoints (between
+      server and clients)
+
+``FLContext`` can be thought of as a Python dictionary that stores key/value pairs. Data items stored in ``FLContext``
+are called properties, or props for short. Props have two attributes: visibility and stickiness.
+
+Visibility
+----------
+Determines whether the prop is only visible to local components that reside in the same process or remote components
+that reside in the peer endpoint:
+    - Props only visible to local components are *private*
+    - Props that are also visible to remote components are *public*.
+
+Stickiness
+----------
+Determines whether the prop is only to be scoped in the current FL Context or if it is to be made
+available to all future FL Contexts:
+    - Props that will become available in all future FL Contexts are *sticky*. This is useful to share objects dynamically created by a component to other components.
+    - Props that are only to be scoped in the current FL Context are *non-sticky*.
+
+.. attention::
+
+    Since public props will be shared with peer endpoints, be very careful about them - they must not be sensitive to
+    privacy! Also since the prop will be sent to the peers through messages, the prop must be serializable.
+
+Accessing Props in FLContext
+----------------------------
+
+To set a prop into the FL Context, use the :meth:`set_prop()<set_prop>` method of FLContext::
+
+    def set_prop(self, key: str, value, private=True, sticky=False)
+
+If the prop already exists in the context, its value is updated with the new value.
+If the prop is not in the context, it is added to the context.
+
+.. note::
+
+    All props have unique names, which must be strings. Once a prop is placed into the FL Context, its attributes can
+    never be changed.
+
+To retrieve a prop, use the get_prop() method::
+
+    def get_prop(self, key, default=None)
+
+To remove a prop from the FL Context, use remove_prop() method::
+
+    def remove_prop(self, key: str)
+
+.. note:: The prop is only removed from the FL Context the remove_prop method is called on.
+
+There are many other useful methods for FL Context. Please see fl_context.py for detail.
+
+.. note::
+
+    Prop names must be strings. Stay away from prop names starting with double underscores "__". These are reserved for
+    the framework. You cannot remove such props from FL context.
+
+Permanent Props in FL Contexts
+------------------------------
+Some props are always added to the FL Context by the framework.
+
+Engine (fl_ctx.get_engine())
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The engine represents the underlying services that can be used by the application. See ServerEngineSpec and/or
+ClientEngineSpec for services they provide.
+
+Run Number (fl_ctx.get_run_number())
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+FL application is always running within a RUN, which has a unique ID number.
+
+Identity Name (fl_ctx.get_identity_name())
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Each running endpoint has a unique identity name. You can get the identity name of the endpoint your application is
+running in.
+
+Peer Context (fl_ctx.get_peer_context())
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+When processing a message from the peer, the FLContext object passed to your callback functions contains the Peer
+Context, which contains the public props from the peer endpoint. Note that not all FL Contexts have peer contexts -
+only those passed to your callback functions (before_task_sent, after_task_sent, and result_received on the Server
+side, and the executor's execute() method on the Client side) have peer contexts. Since filters are invoked to process
+data between the peers, the FLContext passed to filters also contains a peer context.
+
+App Root (fl_ctx.get_prop(FLContextKey.APP_ROOT))
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The name of the current RUN's app folder.
+
+Runtime args (fl_ctx.get_prop(FLContextKey.ARGS))
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Runtime args used to start the server/client process. This is a dict.
+
+Workspace Root (fl_ctx.get_prop(FLContextKey.WORKSPACE_ROOT))
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The name of the workspace root folder.
+
+Workspace Object (fl_ctx.get_prop(FLContextKey.WORKSPACE_OBJECT))
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The workspace object (of :class:`nvflare.apis.workspace.Workspace` type).
+
+Secure Mode (fl_ctx.get_prop(FLContextKey.SECURE_MODE, True))
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Whether NVIDIA FLARE is running in secure mode or not.
+
+
+FL Context Lifecycle
+--------------------
+The NVIDIA FLARE system is a multi-threaded messaging environment. A new FLContext instance is created when processing a new
+message (in the messaging thread). At any time, there could be multiple instances of FL Contexts.
+
+The following is the general lifecycle of a FL Context:
+
+    - **Created**: Never create a new FL Context directly with FLContext()! Always call engine.new_context(). Engine
+      creates a new FL Context with the FL Context Manager :class:`nvflare.api.fl_context.FLContext` (FLCM) associated with the engine. The FLCM keeps the
+      permanent props and all sticky props created by components. When creating a new FL Context, the FLCM copies
+      these props to the created FL Context.
+    - **Used**: The FL Context is used by system and/or application logic - props are read and placed into the context.
+      The FL Context is usually passed to multiple components, and these components can share data thru the FL context
+      passed thru them (one component creates a prop and is used by others).
+    - **Finalized**: Sticky props created/updated during the use of the context are synched back to the FLCM, so that FL
+      contexts created afterwards will contain these props.
+
+Keep in mind that sticky props syncing only happens when the context is finalized, not at the moment that the prop is created.
+
+Data available in the shared FLContext
+--------------------------------------
+The public ``FLContext`` data is sent to other parties along with the ``Shareable`` object.
+It contains the following content:
+
+.. csv-table::
+    :header: Key, Value, Notes
+
+    FLContextKey.PEER_CONTEXT, A dictionary, Its peer's ``FLContext``
+
+.. note::
+
+    For an FL server, its peer is an FL client.
+    For an FL client, its peer is an FL server.
+
+If the role is FL server, the peer context contains the following content:
+
+.. csv-table::
+    :header: Key, Notes
+
+    FLContextKey.CLIENT_NAME, FL client's name
+    AppConstants.CURRENT_ROUND, current training round
+    AppConstants.NUM_STEPS_CURRENT_ROUND, number of steps advanced in the current round
+
+To retrieve the shared ``FLContext`` data:
+
+.. code-block:: python
+
+    shared_context = fl_ctx.get_prop(FLContextKey.PEER_CONTEXT)
+    client_name = shared_context.get_prop(FLContextKey.CLIENT_NAME)
+    current_round = shared_context.get_prop(AppConstants.CURRENT_ROUND)
+    number_steps = shared_context.get_prop(AppConstants.NUM_STEPS_CURRENT_ROUND)
+
+Client Side FL Context
+----------------------
+
+The client-side workflow is a simple loop of:
+    - Ask for the next task to do
+    - Do the task
+    - Submit task result
+
+In each iteration:
+    - A new FLContext instance is created to request the next task from the server.
+    - When a new task assignment is received, the FL context is augmented with the "peer context" received from the server
+    - Task related data is added to the FL context:
+        - Task Name (FLContextKey.TASK_NAME)
+        - Task ID (FLContextKey.TASK_ID)
+        - Task Data (FLContextKey.TASK_DATA)
+    - If any task data filters are configured for this task, the filters are called with this context. Additional props
+      could be added by the filters.
+    - The FL context is then used for the task execution, during which additional props may be added.
+    - The task result (FLContextKey.TASK_RESULT) is added to the FL context.
+    - If any task result filters are configured for this task, the filters are called with this context. Additional
+      props could be added by the filters.
+    - This FL context is used to send task results to the Server. Public props in the context will be sent to the Server.
+    - Finally, this context is finalized - sticky props (if any) are synced with the FLCM.
+
+.. note::
+
+    If any event is fired during the processing, this FLContext instance is also passed to all event handlers, which
+    could create additional props.
+
+The following diagram shows the lifecycle of the FL context for each iteration.
+
+.. image:: resources/FL_Context.png
+    :height: 600px
+
+In the Peer Context, following props from the Server are available:
+
+    - Run Number: peer_ctx.get_run_number())
+
+Server Side FL Context
+----------------------
+
+Task Request Processing
+^^^^^^^^^^^^^^^^^^^^^^^
+When processing the "ask for next task" request from the client, the Server creates a new FL Context. This context is
+used for the task's before_task_sent and after_task_sent callbacks, task data filters (if any), and any event handling
+fired during the task request processing.
+
+Note that this context contains the "peer context" of the client.
+
+In the Peer Context, following props from the Client are available:
+    - Run Number: peer_ctx.get_run_number()
+    - Client Name: peer_ctx.get_identity_name()
+    - May have additional public props
+
+Task Result Processing
+^^^^^^^^^^^^^^^^^^^^^^
+When processing the "submit task result" request from the client, the Server creates a new FL Context. This context is
+used for the task's result_received callback, task result filters (if any), and any event handling fired during the
+task result processing.
+
+Note that this context contains the "peer context" of the client.
+
+In the Peer Context, following props from the Client are available:
+    - Run Number: peer_ctx.get_run_number()
+    - Client Name: peer_ctx.get_identity_name()
+    - Task ID: peer_ctx.get_prop(FLContextKey.TASK_ID)
+    - Task Name: peer_ctx.get_prop(FLContextKey.TASK_NAME)
+    - May have additional public props
+
+Task Done Processing
+^^^^^^^^^^^^^^^^^^^^
+When a task is completed (done normally or timed out), the task's task_done callback (if specified) is called. A new
+FL Context is created at the task completion. This context is used for the task's task_done callback, and any event
+handling fired during the task completion processing.
+
+Note that this context does not contain a peer context.
+
+Controller's Control Flow
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The controller's control_flow method is called at the beginning of the RUN. When the control_flow returns, the RUN is
+completed.
+
+At the beginning of the RUN, a new FL Context is created, and is used to call the control_flow method of the controller.
+This context "lives" until the end of the control loop. This is a long-lived context. This context does not contain a
+peer context.
+
+Peer Context
+------------
+As discussed above, any components can access a FLContext and get/set props from/into it. This allows components running
+within the same process to share data.
+
+The data sharing is not limited to components within the same process (Server or Client). Data props can be shared
+between communicating peers! This is done through what is called "peer context":
+
+    - When the client sends the "ask for next task" request, public props in the client's FLContext are also sent to the
+      Server.
+    - When the Server receives the request and the props from the client, the Server first prepares a new FLContext
+      object X for the request processing, creates another FLContext object to hold the props from the client and sets
+      this FLContext object into X.
+    - All the processing logic has access to X, and can therefore access the peer context via x.get_peer_context() call.
+    - Similarly, during the processing, any involved component can add public props to X, and all such props will be
+      sent back to the Client, which will become content of the "peer context" on the Client side.
+
+What is in the peer context? It depends on what public props are in the FLContext before being sent to the peer. But in
+general, the following props are always available:
+
+    - Run Number (fl_ctx.get_run_number()). This is the ID number of the RUN on the peer site. Note that the peer's RUN
+      number may be different from the host site's RUN number in case the two sites get out of sync.
+    - Identity Name (fl_ctx.get_identity_name()). This is the unique name of the peer site (client name or server name).
+
+Additional props are available from the peer context, depending on the communication scenarios.
+
+.. note:: Peer context should be treated as read-only.
+
+Create ad-hoc FL Context
+------------------------
+All the contexts discussed above are created and managed by the NVIDIA FLARE framework. They should meet most of your
+needs. However, you can create ad-hoc contexts in your program if necessary. To be thread safe, you should create a new
+FLContext instance for each thread - threads should not share the same FLContext object. To create a FLContext object,
+do this::
+
+    ctx = engine.new_context()
+
+As a best practice, when using the context you created, you should use a "with" block so that any sticky props created
+in your processing logic will be synced back to the FLCM when the "with" block exits::
+
+    with ctx:
+        processing logic
+
+API Specification
+-----------------
+
+Property access
+^^^^^^^^^^^^^^^
+.. code-block:: python
+
+    def set_prop(self, key: str, value, private=True, sticky=True)
+
+    def get_prop(self, key, default=None)
+
+With ``set_prop``, you can set any object to the context by providing a key and the value of the object.
+
+When ``private`` is set to True, this property can only be used locally.
+
+When ``private`` is set to False, this property can be shared to its peer during the FL communication.
+
+In other words, when an FL client is communicating with other clients, the non-private properties in the FLContext
+will be sent to those clients.
+
+There are many predefined keys used by different ``FLComponent`` functions. If you want to use the components that
+NVIDIA FLARE provide, please refer to `Data available in the shared FLContext`_ and :mod:`nvflare.apis.fl_constant` for
+details. Otherwise, you can write your own components by extending ``FLComponent``.
diff --git a/docs/programming_guide/shareable.rst b/docs/programming_guide/shareable.rst
new file mode 100644
index 0000000000..b7856c33fc
--- /dev/null
+++ b/docs/programming_guide/shareable.rst
@@ -0,0 +1,82 @@
+Shareable
+=========
+A :class:`Shareable<nvflare.apis.shareable.Shareable>` object represents a communication between server and client.
+Technically a Shareable object is implemented as a Python dict. This dict contains two kinds of information:
+
+Headers
+^^^^^^^
+A special item in the Shareable is "headers", which is a dict itself. The headers are used to carry meta information
+about the communication (e.g. peer identity name, peer's run number, cookies, return code, etc.). Headers are usually
+added and processed by the framework.
+
+Content
+^^^^^^^
+All other items in the Shareable object are the contents of the communication. You can place any elements into the
+Shareable object, but never use ReservedHeaderKey.HEADERS as the key of your content elements.
+
+For all methods of Shareable, see :class:`nvflare.apis.shareable.Shareable`.
+
+Peer Properties
+---------------
+When processing requests or responses (which are all Shareable objects), you can get the information about the peer
+site using::
+
+    peer_props = shareable.get_peer_props()
+
+This is a Python dictionary that contains peer site information.
+
+Cookie
+------
+If you are a workflow developer, when processing the client's Get Task Request, you sometimes want to keep some contextual info with the task assignment
+sent to the client, and expect this info echoed back in the client's task result submission. This can be done through
+the cookie mechanism.
+
+A cookie is just a named piece of data (key/value pair). During the task request processing, any involved component
+(Controller, Filters, Event Handlers, etc.) can add a cookie to the FLContext::
+
+    shareable.add_cookie(name='foo', data=whatEverData)
+
+The headers of the Shareable object can keep a special public prop called "Cookie Jar", which is just a Python dict.
+
+The NVIDIA FLARE framework guarantees that the cookie jar prop will be sent back to the server when the client submits
+the task result to the server.
+
+.. note::
+
+    Cookie data is for the server's consumption only. Client processing logic should not rely on the knowledge of cookie
+    data.
+
+Return Code
+-----------
+Another special header element from the client's task result submission (which is a Shareable object too) is "return code". This is a
+string that indicates whether the task was successfully executed. If this element is not present in the headers, it
+is considered to be successful.
+
+You can get the return code with::
+
+    shareable.get_return_code()
+
+The return code specifies error condition that prevented the task from being executed::
+
+    MISSING_PEER_CONTEXT = "MISSING_PEER_CONTEXT"
+    BAD_PEER_CONTEXT = "BAD_PEER_CONTEXT"
+    RUN_MISMATCH = "RUN_MISMATCH"
+    TASK_UNKNOWN = "TASK_UNKNOWN"
+    TASK_DATA_FILTER_ERROR = "TASK_DATA_FILTER_ERROR"
+    TASK_RESULT_FILTER_ERROR = "TASK_RESULT_FILTER_ERROR"
+    EXECUTION_EXCEPTION = "EXECUTION_EXCEPTION"
+    EXECUTION_RESULT_ERROR = "EXECUTION_RESULT_ERROR"
+
+Some error conditions should never occur (e.g. MISSING_PEER_CONTEXT, BAD_PEER_CONTEXT).
+
+RUN_MISMATCH - client and server are out of sync on the RUN. When this happens, the client will automatically end the RUN.
+
+TASK_UNKNOWN - client cannot find an executor for the assigned task. This is usually caused by misconfiguration of the task table.
+
+TASK_DATA_FILTER_ERROR - client failed to filter task data. Usually bugs in one of the filters.
+
+TASK_RESULT_FILTER_ERROR - client failed to filter results generated by the task executor. Usually bugs in one of the filters.
+
+EXECUTION_EXCEPTION - client failed the task execution. Usually bugs in the executor code.
+
+EXECUTION_RESULT_ERROR - the executor failed to generate a Shareable object as the result - bugs in the code.
diff --git a/docs/quickstart.rst b/docs/quickstart.rst
new file mode 100644
index 0000000000..e0b2c02084
--- /dev/null
+++ b/docs/quickstart.rst
@@ -0,0 +1,16 @@
+##########
+Quickstart
+##########
+
+This section provides a starting point for new users to learn some basic concepts about NVIDIA FLARE.
+Users can work through these examples and gradually get familiar with how NVIDIA FLARE is designed, operates and works.
+
+The learning path of these examples starts with hello_pt (a PyTorch example) and ends with hello_tf2 (a Tensorflow 2 example).
+Each example introduces new concepts about NVIDIA FLARE, while showcasing how some popular libraries and frameworks can
+easily be integrated into the FL process.
+
+.. toctree::
+   :maxdepth: 1
+
+   examples/hello_numpy
+   examples/hello_tf2
diff --git a/docs/resources/Controller.png b/docs/resources/Controller.png
new file mode 100644
index 0000000000..6d2dc61b36
Binary files /dev/null and b/docs/resources/Controller.png differ
diff --git a/docs/resources/Controller_worker.png b/docs/resources/Controller_worker.png
new file mode 100644
index 0000000000..39f7b695e0
Binary files /dev/null and b/docs/resources/Controller_worker.png differ
diff --git a/docs/resources/Executor.png b/docs/resources/Executor.png
new file mode 100644
index 0000000000..665505e894
Binary files /dev/null and b/docs/resources/Executor.png differ
diff --git a/docs/resources/FLAdminAPI_example.py b/docs/resources/FLAdminAPI_example.py
new file mode 100644
index 0000000000..8a48e01ea4
--- /dev/null
+++ b/docs/resources/FLAdminAPI_example.py
@@ -0,0 +1,152 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import argparse
+import time
+
+from nvflare.fuel.hci.client.fl_admin_api import FLAdminAPI
+from nvflare.fuel.hci.client.fl_admin_api_constants import FLDetailKey
+from nvflare.fuel.hci.client.fl_admin_api_spec import TargetType
+
+
+def api_command_wrapper(api_command_result):
+    # prints the result of the command and raises RuntimeError to interrupt command sequence if there is an error
+    print(api_command_result)
+    if not api_command_result["status"] == "SUCCESS":
+        raise RuntimeError("command was not successful!")
+
+    return api_command_result
+
+
+def wait_until_clients_gt2_cb(reply):
+    # use as the callback in wait_until_server_status for waiting until a minimum of 2 clients registered on server
+    if reply["details"][FLDetailKey.REGISTERED_CLIENTS] >= 2:
+        return True
+    else:
+        return False
+
+
+def attempt_login_until_success(api):
+    logged_in = False
+    while not logged_in:
+        reply = None
+        try:
+            print("api.login()")
+            reply = api_command_wrapper(api.login(username="admin@nvidia.com"))
+        except RuntimeError:
+            time.sleep(10)
+            pass
+        if reply and reply["status"] == "SUCCESS":
+            logged_in = True
+    return True
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run_number", type=int, default=100, help="FL run number to start at.")
+    parser.add_argument(
+        "--admin_dir", type=str, default="/workspace/hello_nvflare/packages/admin", help="Path to admin directory."
+    )
+
+    args = parser.parse_args()
+
+    host = ""
+    port = 8003
+
+    # Set up certificate names and admin folders
+    ca_cert = os.path.join(args.admin_dir, "startup", "rootCA.pem")
+    client_cert = os.path.join(args.admin_dir, "startup", "client.crt")
+    client_key = os.path.join(args.admin_dir, "startup", "client.key")
+    upload_dir = os.path.join(args.admin_dir, "transfer")
+    download_dir = os.path.join(args.admin_dir, "download")
+    if not os.path.isdir(download_dir):
+        os.makedirs(download_dir)
+
+    assert os.path.isdir(args.admin_dir), f"admin directory does not exist at {args.admin_dir}"
+    assert os.path.isfile(ca_cert), f"rootCA.pem does not exist at {ca_cert}"
+    assert os.path.isfile(client_cert), f"client.crt does not exist at {client_cert}"
+    assert os.path.isfile(client_key), f"client.key does not exist at {client_key}"
+    run_number = args.run_number
+
+    # Connect with admin client
+    api = FLAdminAPI(
+        host=host,
+        port=port,
+        ca_cert=ca_cert,
+        client_cert=client_cert,
+        client_key=client_key,
+        upload_dir=upload_dir,
+        download_dir=download_dir,
+        debug=False
+    )
+    reply = api.login(username="admin@nvidia.com")
+    for k in reply.keys():
+        assert "error" not in reply[k].lower(), f"Login not successful with {reply}"
+
+    print("api.set_timeout(30)")
+    api_command_wrapper(api.set_timeout(30))
+    apps_to_deploy_and_run = ["hello_numpy_2", "hello_numpy_cross_val"]
+    # With a list of apps_to_deploy_and_run to run consecutively, the assumption is that all apps are configured
+    # properly to run without errors and all apps already exist in upload_dir. Custom code can be written to reconfigure
+    # apps or handle errors that may occur.
+
+    try:
+        for app in apps_to_deploy_and_run:
+            print(f'api.restart("{TargetType.ALL}")')
+            api_command_wrapper(api.restart(TargetType.ALL))
+            attempt_login_until_success(api)
+            print("api.wait_until_server_status(callback=wait_until_clients_gt2_cb)")
+            api_command_wrapper(api.wait_until_server_status(callback=wait_until_clients_gt2_cb))
+            print("api.check_status(TargetType.SERVER)")
+            api_command_wrapper(api.check_status(TargetType.SERVER))
+            print(f"api.set_run_number({run_number})")
+            api_command_wrapper(api.set_run_number(run_number))
+            print(f'api.upload_app("{app}")')
+            api_command_wrapper(api.upload_app(app))
+            print(f'api.deploy_app("{app}", TargetType.ALL)')
+            api_command_wrapper(api.deploy_app(app, TargetType.ALL))
+            print("api.check_status(TargetType.CLIENT)")
+            api_command_wrapper(api.check_status(TargetType.CLIENT))
+            print("api.start_app(TargetType.ALL)")
+            api_command_wrapper(api.start_app(TargetType.ALL))
+            print("api.wait_until_server_stats()")
+            wait_result = api_command_wrapper(api.wait_until_client_status(timeout=2000))
+            if wait_result.get("details"):
+                if wait_result.get("details").get("message"):
+                    if wait_result.get("details").get("message") == "Waited until timeout.":
+                        print("aborting because waited until timeout with clients still not stopped.")
+                        print("api.abort(TargetType.ALL)")
+                        api_command_wrapper(api.abort(TargetType.ALL))
+            # api_command_wrapper("api.wait_until()", api.wait_until())
+            # now server engine status should be stopped
+            print("api.check_status(TargetType.SERVER)")
+            api_command_wrapper(api.check_status(TargetType.SERVER))
+            print("api.check_status(TargetType.CLIENT)")
+            api_command_wrapper(api.check_status(TargetType.CLIENT))
+            time.sleep(10)  # wait for clients to stop in case they take longer than server to stop
+            print("api.check_status(TargetType.CLIENT)")
+            api_command_wrapper(api.check_status(TargetType.CLIENT))
+            run_number = run_number + 1
+
+    except RuntimeError as e:
+        print(f"There was an exception {e}.")
+
+    # log out
+    print("Admin logging out.")
+    api.logout()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/docs/resources/FL_Context.png b/docs/resources/FL_Context.png
new file mode 100644
index 0000000000..38802a4970
Binary files /dev/null and b/docs/resources/FL_Context.png differ
diff --git a/docs/resources/Filters.png b/docs/resources/Filters.png
new file mode 100644
index 0000000000..c66a4eb825
Binary files /dev/null and b/docs/resources/Filters.png differ
diff --git a/docs/resources/Open_Provision_API.png b/docs/resources/Open_Provision_API.png
new file mode 100644
index 0000000000..710290f16d
Binary files /dev/null and b/docs/resources/Open_Provision_API.png differ
diff --git a/docs/resources/config_fed_client.json b/docs/resources/config_fed_client.json
new file mode 100644
index 0000000000..01b636939d
--- /dev/null
+++ b/docs/resources/config_fed_client.json
@@ -0,0 +1,16 @@
+{
+  "format_version": 2,
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "np_trainer.NPTrainer"
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}
\ No newline at end of file
diff --git a/docs/resources/config_fed_server.json b/docs/resources/config_fed_server.json
new file mode 100644
index 0000000000..ff8952bb11
--- /dev/null
+++ b/docs/resources/config_fed_server.json
@@ -0,0 +1,42 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {}
+    }
+  ],
+  "workflows": [
+    {
+      "id": "fed_avg_ctl",
+      "path": "nvflare.app_common.workflows.fed_avg.fed_avg_ctl.FedAvgController",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 10,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 6000
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/docs/resources/fed_avg_round.png b/docs/resources/fed_avg_round.png
new file mode 100644
index 0000000000..06cc3c5d4c
Binary files /dev/null and b/docs/resources/fed_avg_round.png differ
diff --git a/examples/hello-numpy/resources/log.config b/docs/resources/log.config
similarity index 100%
rename from examples/hello-numpy/resources/log.config
rename to docs/resources/log.config
diff --git a/docs/resources/multi_process_executor.png b/docs/resources/multi_process_executor.png
new file mode 100644
index 0000000000..b25792d2b8
Binary files /dev/null and b/docs/resources/multi_process_executor.png differ
diff --git a/docs/resources/provisioning_ui.png b/docs/resources/provisioning_ui.png
new file mode 100644
index 0000000000..6a3482ea84
Binary files /dev/null and b/docs/resources/provisioning_ui.png differ
diff --git a/docs/templates/layout.html b/docs/templates/layout.html
new file mode 100644
index 0000000000..2225bce8a2
--- /dev/null
+++ b/docs/templates/layout.html
@@ -0,0 +1,95 @@
+<!--
+# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+-->
+{% extends "!layout.html" %}
+  {% block sidebartitle %} {{ super() }}
+
+  <style>
+    /* Sidebar header (and topbar for mobile) */
+    .wy-side-nav-search, .wy-nav-top {
+      background: #76b900;
+    }
+
+    .wy-side-nav-search a:link, .wy-nav-top a:link {
+      color: #fff;
+    }
+    .wy-side-nav-search a:visited, .wy-nav-top a:visited {
+      color: #fff;
+    }
+    .wy-side-nav-search a:hover, .wy-nav-top a:hover {
+      color: #fff;
+    }
+
+    .wy-menu-vertical a:link, .wy-menu-vertical a:visited {
+      color: #d9d9d9
+    }
+
+    .wy-menu-vertical a:active {
+      background-color: #76b900
+    }
+
+    .wy-side-nav-search>div.version {
+      color: rgba(0, 0, 0, 0.3)
+    }
+
+    .wy-nav-content {
+      max-width: 80%;
+    }
+
+    .floatleftcol {
+      float: left;
+      max-width: 60%;
+      padding-right: 20px;
+    }
+  </style>
+  {% endblock %}
+
+  {% block footer %} {{ super() }}
+
+  <style>
+  a:link, a:visited {
+    color: #76b900;
+  }
+
+  a:hover {
+    color: #8c0;
+  }
+
+  p {
+    margin-bottom: 1em;
+  }
+
+  .rst-content dl dt {
+    font-weight: unset;
+    margin-bottom: 0;
+  }
+
+  .rst-content .section ul p {
+    margin-bottom: 0px;
+  }
+  </style>
+  {% endblock %}
diff --git a/docs/user_guide.rst b/docs/user_guide.rst
new file mode 100644
index 0000000000..a05fcbd3ac
--- /dev/null
+++ b/docs/user_guide.rst
@@ -0,0 +1,23 @@
+######################################
+User Guide - Provision, Start, Operate
+######################################
+
+This user guide shows how to use NVIDIA FLARE to deploy and operate an FL system on multiple sites with a
+Provision-Start-Operating procedure. A reference application will be used here to show provisioning and basic operation
+of the system through the admin client, but for more details on what you can do with apps with custom components and
+the flexibility that the new NVIDIA FLARE APIs bring, see the :ref:`programming_guide`.
+
+You can also see some example applications integrating with
+`Clara Train <https://docs.nvidia.com/clara/clara-train-sdk/>`_ and
+`MONAI <https://github.com/Project-MONAI/tutorials/tree/master/federated_learning/nvflare>`_
+to see the capabilities of the system and how it can be operated.
+
+.. toctree::
+   :maxdepth: 1
+
+   user_guide/overview
+   user_guide/application
+   user_guide/workspace
+   user_guide/provisioning_tool
+   user_guide/authorization
+   user_guide/admin_commands
diff --git a/docs/user_guide/FLAdminAPI.rst b/docs/user_guide/FLAdminAPI.rst
new file mode 100644
index 0000000000..97d9893bdb
--- /dev/null
+++ b/docs/user_guide/FLAdminAPI.rst
@@ -0,0 +1,620 @@
+FLAdminAPI
+==========
+
+:class:`FLAdminAPI<nvflare.fuel.hci.client.fl_admin_api.FLAdminAPI>` is a wrapper for admin commands that can be issued
+by an admin client to the FL server. You can use a provisioned admin client's certs and keys to initialize an instance
+of FLAdminAPI to programmatically submit commands to the FL server.
+
+Initialization
+--------------
+
+Initialize the API with actual values for the FL setup: host, port, paths to files and directories.
+
+A provisioned admin package should have ca_cert, client_cert, and client_key in the startup folder, and transfer can be
+created at the same level as startup.
+
+Log in with the admin name that corresponds to the provisioned package.
+
+.. code:: python
+
+    from nvflare.fuel.hci.client.fl_admin_api import FLAdminAPI
+    api = FLAdminAPI(
+        host=host,
+        port=port,
+        ca_cert=ca_cert,
+        client_cert=client_cert,
+        client_key=client_key,
+        upload_dir=upload_dir,
+        download_dir=download_dir
+    )
+    api.login(username="admin@nvidia.com")
+
+After using FLAdminAPI, the ``logout()`` function can be called to log out. Both ``login()`` and ``logout()`` are
+inherited from AdminAPI.
+
+.. code:: python
+
+    api.logout()
+
+Usage
+-----
+Simplest sequence to upload, deploy, and start training with the "hello-pt" example app:
+
+.. code:: python
+
+    api.upload_app("hello-pt")
+    api.set_run_number(1)
+    api.deploy_app("hello-pt", "all")
+    api.start_app("all")
+
+Contents of the returned FLAdminAPIResponse can be accessed:
+
+.. code:: python
+
+    reply = api.upload_app("hello-pt")
+    response_text = reply["details"]["message"]
+
+Arguments and targets
+---------------------
+The arguments required for each FLAdminAPI call are specified in the functions section below.
+
+The ``app`` is the name of the directory to upload and deploy containing configurations and custom code.
+
+The ``target`` when needed is where the action should take place. An argument of ``target`` as a string can be a
+singular target of "server" or a specific client name. Where ``target_type`` is required and ``targets`` can
+be an optional list, the call can be submitted to multiple targets:
+
+- If ``target_type`` is "server", the command target is just the server and ``targets`` is ignored
+- If ``target_type`` is "client" and ``targets`` is empty, the command target is all clients
+- If ``target_type`` is "client" and ``targets`` is a list of strings, each a client name, the command target is all clients in the list ``targets``
+- If ``target_type`` is "all" and the command supports it, the command target is the server and all clients
+
+Return Structure
+----------------
+FLAdminAPI calls return an FLAdminAPIResponse dictionary object of key value pairs consisting of a status of type APIStatus, a
+dictionary with details, and in some cases a raw response from the underlying call to AdminAPI (mainly useful for
+debugging). The possible statuses and contents of the details returned are outlined in each of the function calls below.
+
+Note that FLAdminAPI uses the underlying AdminAPI's ``do_command()`` function to submit commands to the server, and you
+can also use this function directly for functions that are not wrapped in an FLAdminAPI function. Returns from AdminAPI
+are included in the FLAdminAPI reply under the "raw" key for some calls and error conditions.
+
+Basic Functions
+---------------
+
+``upload_app(app)``
+^^^^^^^^^^^^^^^^^^^
+Uploads app to upload directory of FL server.
+
++-------+--------------+----------+------------+-------------------------------------------+
+|       | Argument     | Type     | Required   | Description                               |
++=======+==============+==========+============+===========================================+
+| 1     | app          | str      | yes        | name of the folder in upload_dir to upload|
++-------+--------------+----------+------------+-------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful upload of app to server
+* Status ERROR_RUNTIME - app directory may not be located, or other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``set_run_number(run_number)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Sets a run number in order to keep track of the current experiment to deploy and start an app.
+
++-------+------------+-----------+------------+---------------------------------------------+
+|       | Argument   | Type      | Required   | Description                                 |
++=======+============+===========+============+=============================================+
+| 1     | run_number | integer   | yes        | run number to set for the current experiment|
++-------+------------+-----------+------------+---------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful creation of new run number
+* Status PARTIAL_SUCCESS - run number already exists but is now set
+* Status ERROR_SYNTAX - run number must be an integer
+* Status ERROR_RUNTIME - run number may not be changed during training, or other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``deploy_app(app, target_type, targets)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Deploys specified app to run number specific instance on specified targets. Target of "all" deploys to server then
+clients. Run number must be set and app must be uploaded to server already.
+
++-------+-------------+-----------+------------+--------------------------------------------------------------------+
+|       | Argument    | Type      | Required   | Description                                                        |
++=======+=============+===========+============+====================================================================+
+| 1     | app         | str       | yes        | name of app to deploy                                              |
++-------+-------------+-----------+------------+--------------------------------------------------------------------+
+| 2     | target_type | str       | yes        | can be server, client, or all                                      |
++-------+-------------+-----------+------------+--------------------------------------------------------------------+
+| 3     | targets     | list      | no         | list of client names each type str, if target_type is client       |
++-------+-------------+-----------+------------+--------------------------------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful deployment of app (partial success of deployment to some but not all targets is included)
+* Status ERROR_SYNTAX - app and target must be type string
+* Status ERROR_INVALID_CLIENT - server replies that the specified target is invalid
+* Status ERROR_RUNTIME - other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``start_app(target_type, targets)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Starts the currently deployed app at the specified target.
+
++-------+--------------+----------+------------+--------------------------------------------------------------+
+|       | Argument     | Type     | Required   | Description                                                  |
++=======+==============+==========+============+==============================================================+
+| 1     | target_type  | str      | yes        | can be server, client, or all                                |
++-------+--------------+----------+------------+--------------------------------------------------------------+
+| 2     | targets      | list     | no         | list of client names each type str, if target_type is client |
++-------+--------------+----------+------------+--------------------------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful start of app (partial success of some but not all targets is included if server is in targets)
+* Status PARTIAL_SUCCESS - at least one target is already started
+* Status ERROR_SYNTAX - target must be type string
+* Status ERROR_RUNTIME - other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``abort(target_type, targets)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Stops the app at the specified targets. If the target is server and the clients are not stopped, the server will try to
+abort clients first, and the clients may enter cross validation from a status of in training. More functions involving
+abort in the advanced functions section below address some of this.
+
++-------+--------------+----------+------------+--------------------------------------------------------------+
+|       | Argument     | Type     | Required   | Description                                                  |
++=======+==============+==========+============+==============================================================+
+| 1     | target_type  | str      | yes        | can be server or client                                      |
++-------+--------------+----------+------------+--------------------------------------------------------------+
+| 2     | targets      | list     | no         | list of client names each type str, if target_type is client |
++-------+--------------+----------+------------+--------------------------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful abort of app (partial success of some but not all targets is included; abort server may try to abort clients first, and clients may enter cross validation from training and not stop immediately)
+* Status PARTIAL_SUCCESS - training not started
+* Status ERROR_SYNTAX - target must be type string
+* Status ERROR_RUNTIME - other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``shutdown(target_type, targets)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Shuts down the specified targets to stop running FL. The server must be shut down after the clients.
+
++-------+--------------+----------+------------+--------------------------------------------------------------+
+|       | Argument     | Type     | Required   | Description                                                  |
++=======+==============+==========+============+==============================================================+
+| 1     | target_type  | str      | yes        | can be server, client, or all                                |
++-------+--------------+----------+------------+--------------------------------------------------------------+
+| 2     | targets      | list     | no         | list of client names each type str, if target_type is client |
++-------+--------------+----------+------------+--------------------------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful shutdown of app (partial success of some but not all targets is included)
+* Status ERROR_SYNTAX - target must be type string
+* Status ERROR_RUNTIME - other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+Convenience Functions
+---------------------
+
+``check_status(target_type, targets)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Checks and returns the FL status. If target_type is server, the call does not wait for the server to retrieve
+information on the clients but returns the last information the server had at the time this call is made.
+
+If target_type is client, specific clients can be specified in targets, and this call generally takes longer than the
+function to just check the FL server status because this one waits for communication from the server to client then
+back.
+
++-------+--------------+----------+------------+--------------------------------------------------------------+
+|       | Argument     | Type     | Required   | Description                                                  |
++=======+==============+==========+============+==============================================================+
+| 1     | target_type  | str      | yes        | can be server or client                                      |
++-------+--------------+----------+------------+--------------------------------------------------------------+
+| 2     | targets      | list     | no         | list of client names each type str, if target_type is client |
++-------+--------------+----------+------------+--------------------------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful status check (the structure of the details returned depends on what is relevant for the status)\
+* Status ERROR_RUNTIME - other runtime error
+
+With following details structure for target_type server and server statuses training not started and training stopped:
+
++--------------------+----------+-------------------------------------------------------------------+
+| Key                | Type     | Description                                                       |
++====================+==========+===================================================================+
+| run_number         | str      | current set run number, or has not been set                       |
++--------------------+----------+-------------------------------------------------------------------+
+| server_status      | str      | status of the server                                              |
++--------------------+----------+-------------------------------------------------------------------+
+| registered_clients | int      | number of clients that connected and registered on the server     |
++--------------------+----------+-------------------------------------------------------------------+
+| status_table       | list     | table of server's last client statuses                            |
++--------------------+----------+-------------------------------------------------------------------+
+
+With target_type server and server status training started, the details structure is:
+
++--------------------+----------+-------------------------------------------------------------------+
+| Key                | Type     | Description                                                       |
++====================+==========+===================================================================+
+| run_number         | str      | current set run number                                            |
++--------------------+----------+-------------------------------------------------------------------+
+| server_status      | str      | status of the server                                              |
++--------------------+----------+-------------------------------------------------------------------+
+| start_round        | str      | the round for the server to start FL at                           |
++--------------------+----------+-------------------------------------------------------------------+
+| max_round          | str      | the round after which the server will stop FL                     |
++--------------------+----------+-------------------------------------------------------------------+
+| min_num_clients    | int      | minimum number of clients for the server to start aggregation     |
++--------------------+----------+-------------------------------------------------------------------+
+| max_num_clients    | int      | maximum number of clients that can connect to the server          |
++--------------------+----------+-------------------------------------------------------------------+
+| registered_clients | int      | number of clients that connected and registered on the server     |
++--------------------+----------+-------------------------------------------------------------------+
+| submitted_models   | int      | number of clients that submitted models for the current round     |
++--------------------+----------+-------------------------------------------------------------------+
+| status_table       | list     | table of server's last client statuses                            |
++--------------------+----------+-------------------------------------------------------------------+
+
+With target_type client, the details structure is:
+
++--------------------+----------+-------------------------------------------------------------------+
+| Key                | Type     | Description                                                       |
++====================+==========+===================================================================+
+| fl_run_number      | str      | current set run number, or has not been set                       |
++--------------------+----------+-------------------------------------------------------------------+
+| server_status      | str      | status of the server                                              |
++--------------------+----------+-------------------------------------------------------------------+
+| registered_clients | int      | number of clients that connected and registered on the server     |
++--------------------+----------+-------------------------------------------------------------------+
+| status_table       | list     | table of server's last client statuses                            |
++--------------------+----------+-------------------------------------------------------------------+
+
+``get_validation_results(target)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Gets the validation results for the current run number from the server. This is only relevant if cross validation had
+been configured in the app and has been allowed to run, otherwise the results will be empty. If the target is not
+included, it will default to all and this will return the full validation results on the server. The target can also be
+two client names to get just the result of the first client's model on the second client's data.
+
++-------+--------------+----------+------------+------------------------------------------------+
+|       | Argument     | Type     | Required   | Description                                    |
++=======+==============+==========+============+================================================+
+| 1     | target       | str      | no         | can be all, or <client-name1> <client-name2>   |
++-------+--------------+----------+------------+------------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful return of validation results
+* Status ERROR_SYNTAX - target must be type string if included
+* Status ERROR_RUNTIME - other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``restart(target_type, targets)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Restarts the specified targets. Restarting the server will restart all of the clients too. After restarting the server,
+you will need to log in again in order to issue commands.
+
++-------+--------------+----------+------------+--------------------------------------------------------------+
+|       | Argument     | Type     | Required   | Description                                                  |
++=======+==============+==========+============+==============================================================+
+| 1     | target_type  | str      | yes        | can be server, client, or all                                |
++-------+--------------+----------+------------+--------------------------------------------------------------+
+| 2     | targets      | list     | no         | list of client names each type str, if target_type is client |
++-------+--------------+----------+------------+--------------------------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful submission of restart command
+* Status ERROR_SYNTAX - target must be type string
+* Status ERROR_RUNTIME - other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``remove_client(target)``
+^^^^^^^^^^^^^^^^^^^^^^^^^
+Removes the specified targets.
+Note that if a client is removed, you will not be able to issue admin commands through the server to that client
+until the client is restarted (this includes being able to issue the restart command through the API).
+
++-------+--------------+----------+------------+---------------------------------------------+
+|       | Argument     | Type     | Required   | Description                                 |
++=======+==============+==========+============+=============================================+
+| 1     | targets      | list     | yes        | list of client names each type str          |
++-------+--------------+----------+------------+---------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful submission of restart command
+* Status ERROR_SYNTAX - target must be type string
+* Status ERROR_RUNTIME - other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``delete_run_number(run_number)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Deletes the run folder corresponding to the run number on the server and all connected clients. This is not reversible.
+
++-------+------------+-----------+------------+---------------------------------------------+
+|       | Argument   | Type      | Required   | Description                                 |
++=======+============+===========+============+=============================================+
+| 1     | run_number | int       | yes        | run number for the run to delete            |
++-------+------------+-----------+------------+---------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful creation of new run number
+* Status ERROR_SYNTAX - run number must be an integer
+* Status ERROR_RUNTIME - run number may not be changed during training, or other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``set_timeout(timeout)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Sets the timeout for admin commands in seconds. This timeout is the maximum amount of time the server will wait for
+replies from clients. If the timeout is too short, the server may not receive a response because clients may not have a
+chance to reply.
+
++-------+------------+-----------+------------+---------------------------------------------+
+|       | Argument   | Type      | Required   | Description                                 |
++=======+============+===========+============+=============================================+
+| 1     | timeout    | float     | yes        | timeout to set in seconds                   |
++-------+------------+-----------+------------+---------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful setting of timeout
+* Status ERROR_SYNTAX - timeout must be type float
+* Status ERROR_RUNTIME - other runtime error
+
+All with following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``ls_target(target, options, path)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Retrieves the contents of the path (relative to the working directory of the specified target). If no path is specified,
+the contents of the working directory are returned. The target can be "server" or a specific client name for example
+"org2". The allowed options are: "-a" for all, "-l" to use a long listing format, "-t" to sort by modification time
+newest first, "-S" to sort by file size largest first, "-R" to list subdirectories recursively, "-u" with -l to show
+access time otherwise sort by access time.
+
++-------+------------+-----------+------------+---------------------------------------------+
+|       | Argument   | Type      | Required   | Description                                 |
++=======+============+===========+============+=============================================+
+| 1     | target     | str       | yes        | can be server or <client-name>              |
++-------+------------+-----------+------------+---------------------------------------------+
+| 2     | options    | str       | no         | see allowed options above                   |
++-------+------------+-----------+------------+---------------------------------------------+
+| 3     | path       | str       | no         | optional path to a directory                |
++-------+------------+-----------+------------+---------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful return from server
+* Status ERROR_SYNTAX - target must be included and type str
+* Status ERROR_INVALID_CLIENT - server replies that the specified target is invalid
+* Status ERROR_RUNTIME - other runtime error
+
+With following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``cat_target(target, options, file)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Sends the shell command to get the contents of the target's specified file. The target can be "server" or a specific
+client name for example "org2". The file is required and should contain the relative path to the file from the working
+directory of the target. The allowed options are "-n" to number all output lines, "-b" to number nonempty output lines,
+"-s" to suppress repeated empty output lines, and "-T" to display TAB characters as ^I.
+
++-------+------------+-----------+------------+---------------------------------------------+
+|       | Argument   | Type      | Required   | Description                                 |
++=======+============+===========+============+=============================================+
+| 1     | target     | str       | yes        | can be server or <client-name>              |
++-------+------------+-----------+------------+---------------------------------------------+
+| 2     | options    | str       | no         | see allowed options above                   |
++-------+------------+-----------+------------+---------------------------------------------+
+| 3     | file       | str       | yes        | path to the file to return the contents of  |
++-------+------------+-----------+------------+---------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful return from server
+* Status ERROR_SYNTAX - target and file must be included and type str
+* Status ERROR_INVALID_CLIENT - server replies that the specified target is invalid
+* Status ERROR_RUNTIME - other runtime error
+
+With following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``tail_target_log(target, options)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Returns end of target's log allowing for options that the admin client allows: "-n" can be used to specify the
+number of lines for example "-n 100", or "-c" can specify the number of bytes.
+
++-------+------------+-----------+------------+---------------------------------------------+
+|       | Argument   | Type      | Required   | Description                                 |
++=======+============+===========+============+=============================================+
+| 1     | target     | str       | yes        | can be server or <client-name>              |
++-------+------------+-----------+------------+---------------------------------------------+
+| 2     | options    | str       | no         | "-n" for number of lines and "-c" for bytes |
++-------+------------+-----------+------------+---------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful log return
+* Status ERROR_RUNTIME - other runtime error
+
+With following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``grep_target(target, options, pattern, file)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Sends the grep shell command to search the contents of the target's specified file. The target can be "server" or a
+specific client name for example "org2". The file is required and should contain the relative path to the file from the
+working directory of the target. The pattern is also required. The allowed options are "-n" to print line number with
+output lines, "-i" to ignore case distinctions, and "-b" to print the byte offset with output lines.
+
++-------+------------+-----------+------------+---------------------------------------------+
+|       | Argument   | Type      | Required   | Description                                 |
++=======+============+===========+============+=============================================+
+| 1     | target     | str       | yes        | can be server or <client-name>              |
++-------+------------+-----------+------------+---------------------------------------------+
+| 2     | options    | str       | no         | see allowed options above                   |
++-------+------------+-----------+------------+---------------------------------------------+
+| 3     | pattern    | str       | yes        | the pattern to search for                   |
++-------+------------+-----------+------------+---------------------------------------------+
+| 4     | file       | str       | yes        | path to the file to return the contents of  |
++-------+------------+-----------+------------+---------------------------------------------+
+
+Returns:
+
+* Status SUCCESS - successful return from server
+* Status ERROR_SYNTAX - target, pattern, and file must be included and type str
+* Status ERROR_RUNTIME - other runtime error
+
+With following details structure:
+
++--------------+----------+-------------------------------------------------------------------+
+| Key          | Type     | Description                                                       |
++==============+==========+===================================================================+
+| message      | str      | reply string from server                                          |
++--------------+----------+-------------------------------------------------------------------+
+
+``wait_until_(interval, timeout, callback, fail_attempts, **kwargs)``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The functions ``wait_until_server_status()``, ``wait_until_client_status()``, and ``wait_until_server_stats()`` are
+included with the FLAdminAPI in NVIDIA FLARE as examples of useful functions that can be built with other calls in a
+loop with logic. These examples wait until the provided callback returns True, with the option to specify a timeout and
+interval to check the status or stats. There is a default callback to evaluate the reply in the included functions, and
+additional kwargs passed in will be available to the callback. Custom callbacks can be provided to add logic to handle
+checking for other conditions. For these example functions, a timeout should be set in case there are any error
+conditions that result in the system being stuck in a state where the callback never returns True.
+
+You can use the source code of these function as inspiration to create your own functions or logic that makes use of
+other FLAdminAPI calls.
+
++-------+---------------+-----------+------------+---------------------------------------------------------------------+
+|       | Argument      | Type      | Required   | Description                                                         |
++=======+===============+===========+============+=====================================================================+
+| 1     | interval      | int       | no         | interval in seconds between checks of status to provide to callback |
++-------+---------------+-----------+------------+---------------------------------------------------------------------+
+| 2     | timeout       | int       | no         | time in seconds to run before returning with timeout message        |
++-------+---------------+-----------+------------+---------------------------------------------------------------------+
+| 3     | callback      | Callable  | no         | callback to determine condition to end this call and return         |
++-------+---------------+-----------+------------+---------------------------------------------------------------------+
+| 4     | fail_attempts | int       | no         | number of consecutive failed attempts of getting the status         |
++-------+---------------+-----------+------------+---------------------------------------------------------------------+
+
+Questions
+---------
+
+#. Why do I get an error of "Command ___ not found in server or client cmds" even though I did not try any unspecified
+   command?
+
+   The underlying AdminAPI may have not have successfully logged in and obtained a list of available commands to register
+   from the server. Please make sure that the server is accessible and the login is working.
+
+#. Why does the AdminAPI return status APIStatus.SUCCESS even though an error occurred after issuing the command?
+
+   If you send a raw command to the underlying AdminAPI with ``do_command()``, AdminAPI returns APIStatus.SUCCESS if the
+   command was successfully sent to the server and a reply obtained. FLAdminAPI's calls make sense of the underlying
+   server reply and returns a suitable status based on the reply.
+
+#. After a while with the same command, why do I get a SUCCESS from FLAdminAPI but the raw reply contains an error of
+   "not authenticated - no user"?
+
+   The server has a timeout after which ``login()`` must be called again in order for the underlying AdminAPI to be
+   authenticated.
diff --git a/docs/user_guide/admin_commands.rst b/docs/user_guide/admin_commands.rst
new file mode 100644
index 0000000000..da23e766f5
--- /dev/null
+++ b/docs/user_guide/admin_commands.rst
@@ -0,0 +1,66 @@
+##################################
+Admin Client, Commands, FLAdminAPI
+##################################
+
+The FL system is operated by the packages of type admin configured at provisioning. The admin packages contain key and
+certificate files to connect and authenticate with the server, and the administration can be done through an included
+command prompt with ``fl_admin.sh`` or programmatically through the FLAdminAPI.
+
+Admin command prompt
+====================
+After running ``fl_admin.sh``, log in by following the prompt and entering the name of the participant that the admin
+package was provisioned for (or for poc mode, "admin" as the name and password).
+
+Typing "help" or "?" will display a list of the commands and a brief description for each. Typing "? " before a command
+like "? check_status" or "?ls" will provide additional details for the usage of a command. Provided below is a list of
+commands shown as examples of how they may be run with a description.
+
+.. csv-table::
+    :header: Command,Example,Description
+    :widths: 15, 10, 30
+
+    bye,``bye``,Exit from the client
+    help,``help``,Get command help information
+    lpwd,``lpwd``,Print local workspace root directory of the admin client
+    info,``info``,Show folder setup info (upload and download sources and destinations)
+    check_status,``check_status server``,"The FL run number, FL server status, and the registered clients with their names and tokens are displayed. If training is running, the round information is also displayed."
+    ,``check_status client``,"The name, token, and status of each connected client are displayed."
+    ,``check_status client clientname``,"The name, token, and status of the specified client with *clientname* are displayed."
+    upload_app,``upload_app applicationname``,Uploads the application folder to the FL server. Note that *applicationname* is the folder path relative to the "transfer" directory which is at the same level as the "startup" directory containing the script running the admin client.
+    set_run_number,``set_run_number 1``,Creates a folder "run_1" on the server at the same level as the "startup" directory to contain all of the applications for deployment.
+    deploy_app,``deploy_app applicatonname server``,"Deploys the application specified by *applicationname* to the server. Note that *applicationname* is expected to be an application which has been uploaded to the server already and resides in the *transfer* directory on the server (which is at the same level as the *startup* directory by default). *applicationname* can be a relative path if the application is contained in any parent directories, for example apps/segmentation_ct_spleen."
+    ,``deploy_app applicationname client``,Deploys the application specified by *applicationname* to each client. This can also be done per client by specifying a specific client name for this command. Please note that the deployed applications are also in their own workspace named after the run number set by set_run_number above.
+    start_app,``start_app server``,Starts the server training.
+    ,``start_app client``,Starts all of the clients. Individual clients can be started by specifying the client instance name after the start client command.
+    abort,``abort client``,Aborts all of the clients. Individual clients can be aborted by specifying *clientname*. Please note that this may not be instant but may take time for the command to take effect.
+    ,``abort server``,Aborts the server training
+    shutdown,``shutdown client``,Shuts down all of the clients. Individual clients can be shutdown by specifying *clientname*. Please note that this may not be instant but may take time for the command to take effect.
+    ,``shutdown server``,Shuts down the server. Clients must be shut down first before the server is shut down.
+    cat,``cat server startup/fed_server.json -ns``,Show content of a file (-n: number all output lines; -s: suppress repeated empty output lines)
+    ,``cat clientname startup/docker.sh -bT``,Show content of a file (-b: number nonempty output lines; -T: display TAB characters as ^I)
+    env,``env server``,Show environment variables
+    ,``env clientname``,Show environment variables
+    grep,``grep server "info" -i log.txt``,Search for a pattern in a file (-n: print line number; -i: ignore case)
+    head,``head clientname log.txt``,Print the first 10 lines of a file
+    ,``head server log.txt -n 15``,Print the first 15 lines of a file (-n: print the first N lines instead of the first 10)
+    tail,``tail clientname log.txt``,Print the last 10 lines of a file
+    ,``tail server log.txt -n 15``,Print the last 15 lines of a file (-n: output the last N lines instead of the last 10)
+    ls,``ls server -alt``,List files in workspace root directory (-a: all; -l: use a long listing format; -t: sort by modification time)
+    ,``ls clientname -SR``,List files in workspace root directory (-S: sort by file size; -R: list subdirectories recursively)
+    pwd,``pwd server``,Print the name of workspace root directory
+    ,``pwd clientname``,Print the name of workspace root directory
+    validate,``validate src_client dsc_client``, Gets the performance metrics for cross-site validation of a model
+    ,``validate all``, Gets the performance metrics for cross-site validation of all models
+
+
+.. note::
+
+    For multi-GPU training please refer to :ref:`multi-gpu training <multi gpu training>`.
+
+.. tip::
+
+   Outputs of any command can be redirected into a file by using the greater-than symbol ">", however there must be no
+   whitespace before the filename. For example, you may run ``sys_info server >serverinfo.txt``. To only save the
+   file output without printing it, use two greater-than symbols ">>" instead: ``sys_info server >>serverinfo.txt``.
+
+.. include:: FLAdminAPI.rst
diff --git a/docs/user_guide/application.rst b/docs/user_guide/application.rst
new file mode 100644
index 0000000000..6abbca1684
--- /dev/null
+++ b/docs/user_guide/application.rst
@@ -0,0 +1,102 @@
+########################
+NVIDIA FLARE Application
+########################
+
+To upload and run your FL application with NVIDIA FLARE, you need to put required files into an application folder.
+The structure of the folder needs to be::
+
+    app_folder/
+        config/
+            config_fed_client.json
+            config_fed_server.json
+        custom/
+            [any of your custom code].py
+            [another file with custom code].py
+            ...
+        resources/
+            log.config
+
+.. note::
+
+    The configuration JSON files config_fed_server.json and config_fed_client.json may be in the root folder of the FL
+    application or in a sub-folder (for example: config) of the FL application.
+
+***********************
+FL server configuration
+***********************
+
+``config_fed_server.json`` is the FL server configuration file.
+
+Example:
+
+.. literalinclude:: ../resources/config_fed_server.json
+    :language: json
+
+.. csv-table::
+    :header: Key, Notes
+
+    format_version, The NVIDIA FLARE version for this config
+    server, Specify server-specific attributes like heart_beat_timeout for seconds before the heart beat times out
+    task_data_filters, "What filters to apply to data leaving server, see :ref:`NVIDIA FLARE Filters`"
+    task_result_filters, "What filters to apply to data arriving to server, see :ref:`NVIDIA FLARE Filters`"
+    components, All of the Components to use
+    workflows, "What Workflows to use, see :ref:`Controllers and Controller API`
+
+***********************
+FL client configuration
+***********************
+
+``config_fed_client.json`` is the FL client configuration file.
+
+Example:
+
+.. literalinclude:: ../resources/config_fed_client.json
+    :language: json
+
+.. csv-table::
+    :header: Key, Notes
+
+    format_version, The NVIDIA FLARE version for this config
+    executors, The configuration for Tasks and Executors which now includes Trainers
+    task_data_filters, "What filters to apply to data arriving at client, see :ref:`NVIDIA FLARE Filters`"
+    task_result_filters, "What filters to apply to data leaving client, :ref:`NVIDIA FLARE Filters`"
+    components, All of the Components to use
+
+***********
+Custom Code
+***********
+
+You can write your own components following the :ref:`programming_guide:Programming Guide`.
+
+To use it in your application, put the code inside the "custom" folder of the application folder.
+
+In your server or client config, use path to refer to that component.
+
+For example, with a ``SimpleTrainer`` class stored in a file ``my_trainer.py`` inside the custom folder,
+the client config should have the following in order to configure it as an Executor:
+
+.. code-block:: json
+
+    ...
+    "executor": {
+      "path": "my_trainer.SimpleTrainer",
+      "args": {}
+    },
+    ...
+
+.. note::
+
+    Configuration of Executor Tasks is ignored here.
+
+Please follow :ref:`quickstart:Quickstart` to learn more.
+
+*********
+Resources
+*********
+
+A ``log.config`` is needed inside the resources folder.
+This file is for the Python logger to use.
+If you don't want to customize the log behavior, you can use the same ``log.config`` from one of
+the example application folder.
+
+.. literalinclude:: ../resources/log.config
diff --git a/docs/user_guide/authorization.rst b/docs/user_guide/authorization.rst
new file mode 100644
index 0000000000..7664bdf8af
--- /dev/null
+++ b/docs/user_guide/authorization.rst
@@ -0,0 +1,236 @@
+#############
+Authorization
+#############
+
+NVIDIA FLARE implements a role-based authorization framework that determines what a user can or cannot do based on the user’s
+assigned roles configured through AuthPolicyBuilder at provisioning.
+
+********************************
+Terminology for FL authorization
+********************************
+The following concepts are used in defining an authorization policy for NVIDIA FLARE.
+
+Rights
+======
+A right is a permission for a user to do certain things. For example, the right “train_all” allows the user to do
+training for all orgs in a group.
+
+Rules
+=====
+A rule is a policy that an org wants to enforce. For example, the rule “allow_byoc” allows BYOC code to be included in
+the application configurations deployed to the org’s site.
+
+Roles
+=====
+Even though there may be any number of users, they usually are categorized into several types that share the same
+authorization settings. Each such type is called a *role*. A user can be assigned to one or more roles.
+
+Groups
+======
+Even though there could be many orgs in the study, they usually are categorized into several types that share the same
+authorization settings. Each such type is called a *group*. An org can be configured to belong to groups, with a group
+for specifying rules for sites of the org and a group for rights definitions.
+
+***************************
+Define authorization policy
+***************************
+
+Each org can specify its own policies:
+    - Orgs that share the same authorization policies are put in the same group, and authorization policies are defined for the group.
+    - For each group, the permission matrix is defined for role-right combinations.
+    - For each group, permission values are defined for each rule.
+
+The Right Space is a 3D [group, role, right] matrix of permission values, and the Rule Space is a 2D [group, rule] matrix of permission values.
+
+Right Evaluation
+================
+To determine whether a user has a right on a site:
+
+    - Determine the group(s) that the site belongs to
+    - Determine the role(s) of the user
+    - Check the Right Space for each [group, role, right] coordinate. If any point is True, then the result is True. This is what we call the “most generous” policy - as long as any of the user roles has the right in any of the groups the site belongs to, the right is granted. If there is no explicit definition for any point, the default value of the right is taken.
+
+.. note::
+
+   Note that what is important is the user’s role(s). The user’s org is not considered except for deciding if a user is
+   considered “self” for the site. This can in turn affect the right, for example, if site A is with group configured to
+   allow “Operate Self” but not “Operate All” for role “lead_researcher”, a “lead_researcher” user of site A’s org can
+   “Operate” on site A whereas a user that is only “lead_researcher” of another org does not have “Operate” rights.
+
+Rule Evaluation
+===============
+Similar to right evaluation, we also adopt the “most generous” policy to determine the rule value of a site.
+
+Determine the group(s) that the site belongs to
+Check the Rule Space for each [group, rule] coordinate. If any point is True, then the result is True.
+If there is no explicit definition for any point, the default value of the rule is taken.
+
+Defined Rights
+==============
+Currently the following rights are defined:
+
+.. csv-table::
+   :header: Right,Description
+
+    Upload application,whether the user is allowed to upload applications.
+    Deploy All,whether all users of the corresponding role are allowed to deploy applications at sites of a certain group.
+    Deploy Self,whether users of the corresponding role and of the same org as sites of a certain group are allowed to deploy applications.
+    Train All,whether all users of the corresponding role are allowed to perform training actions at sites of a certain group.
+    Train Self,whether users of the corresponding role and of the same org as sites of a certain group are allowed to perform training actions
+    View All,whether all users of the corresponding role are allowed to view information at sites of a certain group.
+    View Self,whether users of the corresponding role and of the same org as sites of a certain group are allowed to view information
+    Operate All,whether all users of the corresponding role are allowed to operate at sites of a certain group.
+    Operate Self,whether users of the corresponding role and of the same org as sites of a certain group are allowed to operate
+
+.. note::
+
+   Rights are always in the context of a group, and note that as mentioned above, for evaluating Rights, the rights
+   group of the site’s org is what is important, and the user’s org is only important when “All” and “Self” have
+   different Rights in that group. Otherwise, the user’s role(s) are all that will matter for determining Rights.
+
+Defined Rules
+=============
+Currently the following rules are defined:
+
+.. csv-table::
+   :header: Rule,Description
+
+    Allow BYOC,whether BYOC code is allowed in application configurations
+    Allow Custom Data List,whether custom data list is allowed in application configurations
+
+.. note::
+
+    For these rules to take effect, the user must provide the implementation of the ``TrainConfigValidator``
+    and edit the ``config_fed_server.json``, otherwise they take no effect.
+
+*************************
+Policy definition example
+*************************
+
+The authorization policy is configured in the authz_policy section of study project YAML file: :ref:`project_yml`.  When
+using the provisioning tool to generate a set of packages, the authorization policy json file is included in the server's
+startup kit zip file.
+
+Here is an example of the generated file::
+
+    {
+     "version": "1.0",
+
+     "roles": {
+       "super": "super user of system",
+       "lead_researcher": "lead researcher of the study",
+       "site_researcher": "site researcher of the study",
+       "site_it": "site IT of the study",
+       "lead_it": "lead IT of the study"
+     },
+     "groups": {
+       "relaxed": {
+         "desc": "the org group with relaxed policies",
+         "rules": {
+           "allow_byoc": true,
+           "allow_custom_datalist": true
+         }
+       },
+       "strict": {
+         "desc": "the org group with strict policies",
+         "rules": {
+           "allow_byoc": false,
+           "allow_custom_datalist": false
+         }
+       },
+       "general": {
+         "desc": "general group user rights",
+         "role_rights": {
+           "super": {
+             "operate_all": true,
+             "view_all": true,
+             "train_all": true
+           },
+           "lead_researcher": {
+             "train_all": true,
+             "view_all": true
+           },
+           "site_researcher": {
+             "train_self": true,
+             "view_self": true
+           },
+           "lead_it": {
+             "operate_all": true,
+             "view_all": true
+           },
+           "site_it": {
+             "operate_self": true,
+             "view_self": true
+           }
+         }
+       }
+     },
+     "users": {
+       "admin@nvidia.com": {
+         "org": "nvidia",
+         "roles": ["super"]
+       },
+       "researcher1@org2.com": {
+         "org": "org2",
+         "roles": ["lead_it", "site_researcher"]
+       },
+       "researcher2@org1.com": {
+         "org": "org1",
+         "roles": ["site_researcher"]
+       }
+     },
+     "orgs": {
+       "org1": ["general", "strict"],
+       "org2": ["general", "relaxed"],
+       "nvidia": ["general"]
+     },
+     "sites": {
+       "org1-a": "org1",
+       "org1-b": "org1",
+       "org2": "org2",
+       "server": "nvidia"
+     }
+    }
+
+A few highlights:
+
+    - Each right has a default value. Default values are used for “holes” in the Right Space.
+    - Each rule has a default value. Default values are used for “holes” in the Rule Space.
+    - Each user is assigned to a single org and one or more roles;
+    - Each site is assigned a single org;
+    - Each org is assigned to one or more groups;
+    - In each group, a rule and/or right matrix is defined.
+
+****************************
+Admin command authorizations
+****************************
+
+Each command from the admin user is subject to authorization. The command is executed only if the authorization is passed.
+
+Commands are grouped into the following action groups for rights:
+
+UPLOAD - uploading application configuration to the server.
+===========================================================
+Command(s) in this group require the “upload application" right.
+Furthermore, if the application contains BYOC code, the site’s “allow_byoc” must be true.
+Furthermore, if the application contains a custom data list, the site’s “allow_custom_datalist” must be true.
+
+DEPLOY - deploy the application to a site
+=========================================
+Command(s) in this group require the “deploy all” or “deploy self” right.
+Furthermore, if the application contains BYOC code, the site’s “allow_byoc” must be true.
+Furthermore, if the application contains a custom data list, the site’s “allow_custom_datalist” must be true.
+
+TRAIN - training related actions (set run, start/abort training)
+================================================================
+Command(s) in this group require the “train all” or “train self” right.
+
+VIEW - view training and/or system info (ls, head, tail, grep, pwd, …)
+================================================================================
+Command(s) in this group require the “view all” or “view self” right.
+
+OPERATE - application operation (shutdown, restart server/clients, sys_info)
+============================================================================
+Command(s) in this group require the “operate all” or “operate self” right.
+
+
diff --git a/docs/user_guide/overview.rst b/docs/user_guide/overview.rst
new file mode 100644
index 0000000000..86f90689b3
--- /dev/null
+++ b/docs/user_guide/overview.rst
@@ -0,0 +1,304 @@
+########
+Overview
+########
+
+************
+Introduction
+************
+
+NVIDIA FLARE utilizes provisioning and admin client to reduce the amount of human coordination involved to set up a federated learning project
+and provides an admin the ability to deploy the server and client configurations, start the server / clients, abort the training,
+restart the training, and more. A provisioning tool can be configured to create one startup kit for each site in an encrypted package.
+These packages can then be delivered to each site ready to go, streamlining the process to provision, start, and operate federated learning.
+
+Provision - Start - Operate
+===========================
+
+Provision
+---------
+Lead IT generates the packages for the server / clients / admins, protected with passwords
+
+Start
+-----
+Site IT each installs their own packages, starts the services, and maps the data location
+
+Operate
+-------
+Lead scientists / administrators control the federated learning process: deploy application, check statuses, start / abort / shutdown training
+
+
+******************************************************************************
+Provision: Configure and generate packages for the server, clients, and admins
+******************************************************************************
+
+One party leads the process of configuring the provisioning tool and using it to generate startup kits for each party in
+the federated learning training project:
+
+Preparation for using the provisioning tool
+===========================================
+
+After :ref:`installation`, the provisioning tool is available via ``provision`` command.
+
+Provisioning a federated learning project
+=========================================
+The :ref:`user_guide/provisioning_tool:Provisioning tool` page has details on the contents of the
+provisioning tool. Edit the :ref:`user_guide/provisioning_tool:Project yaml file` in the directory with the provisioning tool to meet your
+project requirements, then run the startup kit with (here we assume your project.yml is in current working directory)::
+
+    provision -p project.yml
+
+A directory named "packages" containing each of the generated zip files is created where provision.py is run.
+The console displays a list of zip files and their passwords. We suggest you copy the console output
+and "packages" folder to a safe location. The passwords shown below are for demonstration purposes only::
+
+    Project yaml file: /home/nvflare-venv/project.yml.
+    ┏━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┓
+    ┃ participant           ┃ org    ┃ destination               ┃ password         ┃
+    ┡━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━┩
+    │ localhost             │ nvidia │ localhost.zip             │ Dby2BhwJdLKfStWl │
+    │ org1-a                │ org1   │ org1-a.zip                │ BlLXFKgWp0Qu28cS │
+    │ org1-b                │ org1   │ org1-b.zip                │ Lu6w0aCf1RhAqSlJ │
+    │ org2                  │ org2   │ org2.zip                  │ KdotOafkUl2ziRh5 │
+    │ org3                  │ org3   │ org3.zip                  │ t05cHjnd4WNSo62x │
+    │ admin@nvidia.com      │ nvidia │ admin@nvidia.com.zip      │ eyTrthBudD7noW1s │
+    │ researcher@nvidia.com │ nvidia │ researcher@nvidia.com.zip │ s52b8h9ToIRuALpx │
+    │ researcher@org1.com   │ org1   │ researcher@org1.com.zip   │ dmlt3ySsAeU0V9F7 │
+    │ researcher@org2.com   │ org2   │ researcher@org2.com.zip   │ GJS6eb410q0ijlCZ │
+    │ it@org2.com           │ org2   │ it@org2.com.zip           │ s3lYvaL2tqX0Wrjb │
+    └───────────────────────┴────────┴───────────────────────────┴──────────────────┘
+
+.. tip:: You can run ``provision -n -p project.yml`` to generate zip files without password protection.
+
+.. tip:: For security reasons, it is recommended to send the password to each participant separately from the package itself.
+
+After generating packages: Distribute and extract
+=================================================
+Please let each participant know that the packages are password protected. In Ubuntu, the following command can be used
+to extract the packages::
+
+    unzip -P $PASSWORD $ZIP_FILE -d $DIRECTORY_TO_EXTRACT_TO
+
+Using ``-d $DIRECTORY_TO_EXTRACT_TO`` is optional, and without it, a "startup" folder will be extracted to the current
+directory the package is in. Either way, the parent folder containing this "startup" folder (*$DIRECTORY_TO_EXTRACT_TO*
+if the ``-d`` option was used) will be the server, client, or admin client workspace root directory, and the party
+running the package will need write access there.
+
+.. note::
+
+   It is important that this "startup" folder is not renamed because the code relies upon this for operation. Please
+   note that a "transfer" directory and deployed applications will be created at the level of this "startup" folder. See the
+   section on `Internal folder and file structures for NVIDIA FLARE`_ below for more details.
+
+************************************************************************************
+Start: Instructions for each participant to start running FL with their startup kits
+************************************************************************************
+
+.. attention:: Please always safeguard .key files!
+
+Federated learning server ($SERVER_NAME.zip)
+============================================
+One single server will coordinate the federated learning training and be the main hub all clients and administrator
+clients connect to.
+
+After unzipping the package server.zip, run the start.sh file from the "startup" folder you unzipped to start the server.
+
+The rootCA.pem file is pointed to by "ssl_root_cert" in fed_server.json.  If you plan to move/copy it to a different place,
+you will need to modify fed_server.json.  The same applies to the other two files, server.crt and server.key.
+
+.. note::
+
+   When launching the FL server inside a docker with ``docker run``, use ``--net=host`` to map hostname into that
+   docker instance.  In secure gRPC, FL server has to bind the hostname specified in the provisioning stage.
+   Always make sure that hostname is what FL server can bind to. Additionally,
+   the port that the server communicates on must also not be blocked by any firewalls.
+
+If clients from other machines cannot connect to the server, make sure that the host name specified when generating
+the startup kits in the provisioning process resolves to the correct IP. In ubuntu, an entry may need to be added to
+/etc/hosts with the IP and the host name.
+
+Federated learning client ($CLIENT_NAME.zip)
+============================================
+Each site participating in federated learning training is a client. Each package for a client is named after the client
+name specified when provisioning the project.
+
+After unzipping the package (for details see `After generating packages: Distribute and extract`_), run ``start.sh``
+from the "startup" folder you unzipped to start the client.
+
+.. tip::
+
+   You need to first install NVIDIA FLARE package before running the ``start.sh`` shell script.  NVIDIA FLARE is available
+   on PyPi and can be installed with ``python3 -m pip install nvflare``.
+
+   Depending on the deployed application which shall start later, your environment may need some additional
+   Python packages.  If you haven't installed them, do it after you install NVIDIA FLARE.  NVIDIA FLARE does not dictate
+   your deep learning environments.  It's completely up to you to set it up.
+
+.. note::
+
+    Coordination for where to mount the data may be needed depending on where the dataset is located in the application to be deployed.
+
+The rootCA.pem file is pointed to by "ssl_root_cert" in fed_client.json.  If you plan to move/copy it to a different place,
+you will need to modify fed_client.json.  The same applies to the other two files, client.crt and client.key.
+
+The client name in your submission to participate this federated learning project is embedded in the CN field of client
+certificate, which uniquely identifies the participant. As such, please safeguard its private key, client.key.
+
+When a client successfully connects to the FL server, the server and that client will both log a token confirming that
+the client successfully connected:
+
+Server::
+
+    2020-07-07 03:48:49,712 - ClientManager - INFO - Client: New client abcd@127.0.0.1 joined. Sent token: f279157b-df8c-aa1b-8560-2c43efa257bc.  Total clients: 1
+
+Client::
+
+    2020-07-07 03:48:49,713 - FederatedClient - INFO - Successfully registered client:abcd for exampletraining. Got token:f279157b-df8c-aa1b-8560-2c43efa257bc
+
+If a connection cannot be made, the client will repeatedly try to connect and for each failure log::
+
+    Could not connect to server. Setting flag for stopping training. failed to connect to all addresses
+
+If the server is up, you may need to troubleshoot with settings for firewall ports to make sure that the proper
+permissions are in place. This could require coordination between the lead IT and site IT personnel.
+
+Federated learning administration client ($EMAIL.zip)
+=====================================================
+Each admin client will be able to connect and submit commands to the server. Each admin client package is named after
+the email specified when provisioning the project, and the same email will need to be entered for authentication when
+the admin client is launched.
+
+Install the wheel package first with::
+
+    python3 -m pip install nvflare
+
+
+After installation, you can run the **fl_admin.sh** file to start communicating to the FL server.
+The FL server must be running and there must be a successful connection between the admin
+client and the FL server in order for the admin client to start. For the prompt **User Name:**, enter the email that was
+used for that admin client in the provisioning of the project.
+
+The rootCA.pem file is pointed to by "ca_cert" in fl_admin.sh.  If you plan to move/copy it to a different place,
+you will need to modify the corresponding script.  The same applies to the other two files, client.crt and client.key.
+
+The email to participate this FL project is embedded in the CN field of client certificate, which uniquely identifies
+the participant. As such, please safeguard its private key, client.key.
+
+.. attention::
+
+   You will need write access in the directory containing the "startup" folder because the "transfer" directory for
+   uploading files as well as directories created for federated learning runs will live here. For details, see
+   `Internal folder and file structures for NVIDIA FLARE`_.
+
+*******************************************************
+Operate: Running federated learning as an administrator
+*******************************************************
+
+Running federated learning from the administration client
+=========================================================
+With all connections between the FL server, FL clients, and administration clients open and all of the parties
+started successfully as described in the preceding section, `Federated learning administration client ($EMAIL.zip)`_,
+admin commands can be used to operate a federated learning project. The FLAdminAPI provides a way to programmatically
+issue commands to operate the system so it can be run with a script.
+
+For a complete list of admin commands, see :ref:`user_guide/admin_commands:Administrator commands`.
+
+For examples of using the commands to operate a FL system, see the examples in the :ref:`quickstart` section.
+
+****************************************************
+Internal folder and file structures for NVIDIA FLARE
+****************************************************
+
+Server side folder and file structure
+=====================================
+::
+
+    /some_path_on_fl_server/fl_server_workspace_root/
+        admin_audit.log
+        log.txt
+        startup/
+            authorization.json
+            fed_server.json
+            log.config
+            readme.txt
+            rootCA.pem
+            server_context.tenseal
+            server.crt
+            server.key
+            signature.pkl
+            start.sh
+            stop_fl.sh
+            sub_start.sh
+        transfer/
+        run_1/
+            mmar_server/
+                config/
+                models/
+                resources/
+            mmar_client1/
+                config/
+                models/
+                resources/
+            mmar_client2/
+                config/
+                models/
+                resources/
+            ...
+            cross_validation/
+        run_2/
+            ......
+
+Client side folder and file structure
+=====================================
+::
+
+    /some_path_on_fl_client/fl_client_workspace_root/
+        log.txt
+        startup/
+            client_context.tenseal
+            client.crt
+            client.key
+            fed_client.json
+            log.config
+            readme.txt
+            rootCA.pem
+            signature.pkl
+            start.sh
+            stop_fl.sh
+            sub_start.sh
+        transfer/
+        run_1/
+            mmar_client1/
+                config/
+                cross_validation/
+                models/
+                resources/
+        run_2/
+            mmar_client1/
+                config/
+                cross_validation/
+                models/
+                resources/
+        run_3/
+            ......
+
+Administrator side folder and file structure
+============================================
+::
+
+    /some_path_on_fl_admin/fl_administrator_workspace_root/
+        startup/
+            client.crt
+            client.key
+            fl_admin.sh
+            readme.txt
+            rootCA.pem
+            signature.pkl
+        transfer/
+            application_for_uploading/
+                config/
+                models/
+                resources/
+            application2_for_uploading/
+                config/
+                models/
+                resources/
diff --git a/docs/user_guide/provisioning_tool.rst b/docs/user_guide/provisioning_tool.rst
new file mode 100644
index 0000000000..7ec5671462
--- /dev/null
+++ b/docs/user_guide/provisioning_tool.rst
@@ -0,0 +1,417 @@
+############################
+Provisioning in NVIDIA FLARE
+############################
+
+The purpose of provisioning in NVIDIA FLARE is to generate mutual-trusted system-wide configurations for all participants
+so all of them can join the NVIDIA FLARE system across different locations. The mutual-trust is a mandatory feature of the
+Open Provision API as every participant authenticates others by the information inside the configuration. The
+configurations usually include, but are not limited to, the following information:
+
+    - network discovery, such as domain names, port numbers or IP addresses
+    - credentials for authentication, such as certificates of participants and root authority
+    - authorization policy, such as roles, rights and rules
+    - tamper-proof mechanism, such as signatures
+    - convenient commands, such as shell scripts with default command line options to easily start an individual participant
+
+In order to enable developers to freely add / modify / remove the above configurations to fit their own requirements,
+we created the Open Provision API. Now developers can take advantage of this API to perform the provisioning tasks
+which meet their own requirements in addition to the default provisioning before for creating packages for the
+server, client, and administrators.
+
+******************
+Open Provision API
+******************
+
+Architecture
+============
+
+.. image:: ../resources/Open_Provision_API.png
+    :height: 350px
+
+The above diagram describes the architecture of Open Provision API in blue.  Those two green blocks are the sample
+python code (provision.py) collecting information (project.yml) and interacting with components of Open Provision API to
+accomplish one provisioning task.  The Provisioner and blocks inside the blue box are classes or subclasses of Open
+Provision API.
+
+Details
+=======
+
+project.yml
+-----------
+This is a simple yaml file, describing participants and builders.  Note that Open Provision API itself does not define
+the format of this file.  Any developer can create his/her own file that describes participants and builders in a
+different format.  The developer can even store such information in one URL as long as there is code
+(provision.py in the above sample diagram) that can load the information and convert such information to calls
+to Open Provision API.
+
+provision.py
+------------
+This python file is the sample application to interact with the Open Provision API.  It also loads project.yml, parses
+command line options, instantiates classes/subclasses defined from Open Provision API and displays helpful messages to users.
+As mentioned previously, developers are encouraged to modify provision.py or write their own applications that fit their own requirements.
+It is also possible to completely utilize Open Provision API without any standalone applications.  For example, if
+developers have their existing applications and would like to add provisioning capabilities for the NVIDIA FLARE system,
+they can add API calls to Open Provision API to generate required outputs.
+
+Provisioner
+-----------
+This is the container class that owns all instances of Study, Workspace, Provision Context, Builders and Participants,
+as shown in the above diagram.  A typical usage of this class is like the following::
+
+    provisioner = Provisioner(workspace_full_path, builders)
+
+    provisioner.provision(study)
+
+Study
+-----
+The Study class keeps information about participants.  Therefore, information of any participant can be retrieved from
+the Study instance::
+
+    class Study(object):
+       def __init__(self, name: str, description: str, participants: List[Participant]):
+           self.name = name
+           all_names = list()
+           for p in participants:
+               if p.name in all_names:
+                   raise ValueError(f"Unable to add a duplicate name {p.name} into this study.")
+               else:
+                   all_names.append(p.name)
+           self.description = description
+           self.participants = participants
+
+       def get_participants_by_type(self, type, first_only=True):
+           found = list()
+           for p in self.participants:
+               if p.type == type:
+                   if first_only:
+                       return p
+                   else:
+                       found.append(p)
+           return found
+
+Participant
+-----------
+Each participant is one entity that communicates with other participants inside the NVIDIA FLARE system during runtime.
+Each participant has the following attributes: type, name, org and props.  The attribute ``props`` is a dictionary and
+stores additional information::
+
+    class Participant(object):
+       def __init__(self, type: str, name: str, org: str, *args, **kwargs):
+           self.type = type
+           self.name = name
+           self.org = org
+           self.subject = name
+           self.props = kwargs
+
+The name of each participant must be unique.  This is enforced in Study's __init__ method.  The type
+defines the behavior of this participant when it is alive in the NVIDIA FLARE system.  For example, type = 'server' defines
+that the participant acts as a server.  Three types are commonly used for a typical NVIDIA FLARE system: server, client, and
+admin.  However, developers can freely add other types when needed, such as 'gateway,' 'proxy,' or 'database.'  The
+builders can take such information into consideration so that they can generate relevant results based on the type
+attribute.
+
+Builder
+-------
+The builders in the above diagram are provided as a convenient way to generate commonly used zip files for a typical
+NVIDIA FLARE system.  Developers are encouraged to add / modify or even remove those builders to fit their own requirements.
+
+Each builder is responsible for taking the information from study, its own __init__ arguments, and provisioner to
+generate data.  For example, the HEBuilder is responsible for generating tenseal context files for server and client,
+but not admin.  Additionally, the context for servers does not include either public key or secret key while the
+context for clients include both.  Its __init__ arguments consist of poly_modules_degree, coeff_mod_bit_sizes,
+scale_bits and scheme.  With all of the information, HEBuilder can output context files correctly.
+
+Provisioner calls each builder's initialize method first during provisioning time in a loop.  This allows builders to
+prepare information and to populate their instance variables.  After calling each builder's initialize method, the
+Provisioner calls each builder's build method in another loop.  This method is usually implemented to execute the
+actual build process (generating necessary files).  At the end, the provisioner calls the finalize method of each
+builder in REVERSE ORDER in the third loop so all builders have a chance to wrap up their states.  This comes from
+the convention that the earlier a builder's initialize method is called, the later its finalize method should be called.
+
+The iterations in each of the above three loops are always determined by the builders list, the second argument
+passed to Provisioner class.  Therefore, different orders in the builders list affect the results.
+
+For example, when one builder's finalize method cleans up and removes the wip folder which is shared by all builders,
+builders being called after it will not be able to access the wip folder.
+
+.. note:: The collaboration among all builders is the responsibility of Open Provision API developers.
+
+Every builder has to subclass the Builder class and override one or more of these three methods::
+
+    class Builder(ABC):
+       def initialize(self, ctx: dict):
+           pass
+
+       def build(self, study: Study, ctx: dict):
+           pass
+
+       def finalize(self, ctx: dict):
+           pass
+
+Workspace
+---------
+Each builder can access four folders under provision workspace which is managed by Provisioner (see Provisioner's
+first argument).  Those folders are 'wip' (for working-in-progress), 'kit_dir' (a subfolder in 'wip'), 'state' (used
+to persist information between different revisions) and 'resources' (for read-only / static information).
+
+Provision Context
+-----------------
+Provision context is created by Provisioner and can be read / written by all participants and builders.  A builder
+might add a piece of information to it so that another builder can retrieve it.  As a hypothetical example, developers
+might want to add a second homomorphic encryption builder to generate a different set of HE contexts based on
+certificates from CertBuilder and the context of the first HE builder.  To achieve this, the developers can write
+certificates to provision context at CertBuilder and he context to provision context at HEBuilder.  The information
+is automatically available to the second HE builder.
+
+Open Provision API Case Studies
+===============================
+Before we start, please remember that the builders have three methods to be implemented optionally, initialize, build
+and finalize.  The Provisioner calls initialize methods of all builders, then build methods of all builders.  Both in
+the order of builders list.  However, the finalize methods of all builders are called by Provisioner in REVERSE order.
+Please keep this in mind.
+
+For example, in Case 2, the builders is a list and append method adds the WebPostDistributionBuilder to the end of the
+builder list.  As mentioned above, the initialize and build methods are called in the order of the builder list while
+the finalize method is called in the reverse order.  We can expect the finalize method of the WebPostDistributionBuilder
+is called before other builders' finalize methods and before other builders' build methods.
+
+Case 1: generating additional files
+-----------------------------------
+The developers would like to add a configuration file about a database server to admin participants.  The configuration
+is like this::
+
+    [database]
+    db_server = server name
+    db_port = port_number
+    user_name = admin's name
+
+As this requires adding one file to every admin participant, the developer can write a DBBuilder as follows::
+
+    class DBConfigBuilder(Builder):
+       def __init__(self, db_server, db_port):
+           self.db_server = db_server
+           self.db_port = db_port
+
+       def build(self, study, ctx):
+           for admin in study.get_participants_by_type("admin", first_only=False):
+               dest_dir = self.get_kit_dir(admin, ctx)
+               with open(os.path.join(dest_dir, "database.conf"), 'wt') as f:
+                   f.write("[database]\n")
+                   f.write(f"db_server = {self.db_server}\n")
+                   f.write(f"db_port = {self.db_port}\n")
+                   f.write(f"user_name = {admin.name}\n")
+
+And in project.yml, add an entry in the builders section::
+
+    - path: byob.DBConfigBuilder
+      args:
+        db_server: example.com
+        db_port: 5432
+
+Case 2: enhancing an existing builder
+-------------------------------------
+The developer would like to push the generated zip file, done by nvflare.lighter.impl.workspace.DistributionBuilder, to
+a web server via a POST method.  This can be done easily by replacing the DistributionBuilder with a new builder as
+follows (after pip install requests)::
+
+    class WebPostDistributionBuilder(Builder):
+       def __init__(self, url):
+           self.url = url
+
+       def build(self, study: Study, ctx: dict):
+           wip_dir = self.get_wip_dir(ctx)
+           dirs = [name for name in os.listdir(wip_dir) if os.path.isdir(os.path.join(wip_dir, name))]
+           for dir in dirs:
+               dest_zip_file = os.path.join(wip_dir, f"{dir}")
+               shutil.make_archive(dest_zip_file, "zip", root_dir=os.path.join(wip_dir, dir), base_dir="startup")
+               files = {"upload_file": open(dest_zip_file, "rb")}
+               r = requests.post(self.url, files=files)
+
+And just replace the existing one with the new builder under Builders in the project.yml::
+
+    - path: byob.WebPostDistributionBuilder
+      args:
+        url: https://example.com/nvflare/provision
+
+For the above two cases, if developers opt to use Open Provision API directly instead of project.yml, they can do this
+(some code omitted for clarity)::
+
+    from byob import WebPostDistributionBuilder
+    builders = list()
+    # Adding other builders
+    # ...
+
+    # Using our new WebPostDistributionBuilder builders.append(WebPostDistributionBuilder(url="https://example.com/nvflare/provision"))
+
+    # Instantiate Provisioner
+    provisioner = Provisioner(workspace_full_path, builders)
+
+Case 3: adding both new builders and participants of new types
+--------------------------------------------------------------
+The developers would like to add participants of type = 'gateway.'  In order to handle this type of participants, a new
+builder is needed to write gateway specific configuration.  First, specify that in project.yml::
+
+    - name: gateway1
+      type: gateway
+      org: nvidia
+      port: 8102
+
+or in API style::
+
+    participants = list()
+    p = Participant(name="gateway1", type="gateway", org="nvidia", port=8102)
+    participants.append(p)
+
+A new builder to write 'gateway.conf' can be implemented as follows (for reference)::
+
+    class GWConfigBuilder(Builder):
+      def build(self, study, ctx):
+          for gw in study.get_participants_by_type("gateway", first_only=False):
+              dest_dir = self.get_kit_dir(gw, ctx)
+              with open(os.path.join(dest_dir, "gateway.conf"), 'wt') as f:
+                  port = gw.props.get("port")
+                  f.write("[gateway]\n")
+                  f.write(f"name = {gw.name}\n")
+                  f.write(f"port = {port}\n")
+
+Takeaways for Custom Builders
+-----------------------------
+From the cases shown previously, implementing your own Builders only requires the following steps:
+
+#. Subclass the Builder class
+#. Implement the required methods (initialize, build, finalize).  Not all of them have to be implemented.
+#. The builder can locate the working-in-progress space from the return value of this method self.get_wip_dir(ctx).  This
+   space is shared by all builders.
+#. Builder writes participant-specific files to the kit directory which is the return value of self.get_kit_dir(participant, ctx)
+#. Builders have to coordinate with one another.  For example, the WebPostDistributionBuilder generates zip files from the
+   contents inside kit directories.  That implies some other builders have to write those contents first.
+
+Bundled builders
+================
+The following is the list of bundled builders included by default in the NVIDIA FLARE package.  They are provided as a convenient tool.  As
+mentioned previously, developers are encouraged to add / modify / remove builders based on their own requirements:
+    - CertBuilder
+    - HEBuilder
+    - SignatureBuilder
+    - TemplateBuilder
+    - StaticFileBuilder
+    - WorkspaceBuilder
+    - DistributionBuilder
+    - AuthPolicyBuilder
+
+::
+
+    workspace structure
+    └── example_project
+        ├── prod_00
+        │   ├── admin@nvidia.com
+        │   │   └── startup
+        │   ├── localhost
+        │   │   └── startup
+        │   ├── site1
+        │   │   └── startup
+        │   └── site2
+        │       └── startup
+        ├── prod_01
+        │   ├── admin@nvidia.com
+        │   │   └── startup
+        │   ├── localhost
+        │   │   └── startup
+        │   ├── site1
+        │   │   └── startup
+        │   └── site2
+        │       └── startup
+        ├── prod_02
+        │   ├── admin@nvidia.com
+        │   │   └── startup
+        │   ├── localhost
+        │   │   └── startup
+        │   ├── site1
+        │   │   └── startup
+        │   └── site2
+        │       └── startup
+        ├── resources
+        └── state
+
+The prod_NN folders contain the provisioning results.  The number, NN, increases every time the provision command runs successfully.
+
+*****************
+Project yaml file
+*****************
+
+This is the key file that describes the information which provisioning tool will be using to generate startup kits for server, clients and admins.
+If there is no ``project.yml`` in your current working directory, simply run ``provision`` without any option.  It
+will ask you if you would like to have one sample copy of this file created.
+
+.. code-block:: shell
+
+  (nvflare-venv) ~/workspace$ provision
+  No project.yml found in current folder.  Is it OK to generate one at /home/nvflare/workspace/project.yml for you? (y/N) y
+  /home/nvflare/workspace/project.yml was created.  Please edit it to fit your FL configuration.
+
+
+..  Edit the project.yml configuration file to meet your project requirements:
+
+    - "api_version" must be 2 for current release of provisioning tool
+    - "he_config" describes the Homomorphic Encryption configuration
+    - "config_folder" is the folder name that contains configuration information
+    - "disable_authz" is a boolean to instruct FL server whether to enforce role-based rights on admin users. true means all admin users are in role super.
+    - "name" is used to identify this project.
+    - "server" section describes the FL server.
+        - "org" is for the name of the owner of this server.
+        - "cn" is the "Fully Qualified Domain Name" and it is very important that this is correct. If this information is not completely correct, the security handshake between the server and clients will fail. Please note that this cannot just be an IP address.
+        - "fed_learn_port" is the port number for communication between the FL server and FL clients
+        - "admin_port" is the port number for communication between the FL server and FL administration client
+        - "admin_storage" is directory name, related to the WORKSPACE, to store files by admin process on server
+        - "min_num_clients" is the minimum number of clients for federated learning to begin
+        - "max_num_clients" is the maximum number of clients allowed in this instance of federated learning
+        - "config_validator" is the class that validates if the uploaded app has correct syntax
+    - The "fl_clients" section describes the FL clients, with one "org", "client_name", and "email" for each client. Please note that each "client_name" must be unique. It will show in the admin console.
+    - The "admin_clients" section describes the FL admin clients. The "email" for each must be unique.
+    - The "authz_policy" section describes the policy imposed on organizations and roles.  Roles of each organization have specific rights.  Sites of each
+    organization have specific rules.  Rights grant roles to do certain operations.  Rules allow certain operations performed in sites.
+
+.. attention:: Please make sure that the FL server port number is accessible by all participating sides.
+
+
+.. _project_yml:
+
+Default project.yml file
+========================
+
+The following is an example of the default project.yml file.
+
+.. literalinclude:: ../../nvflare/lighter/project.yml
+  :language: yaml
+
+*****************************
+Provision commandline options
+*****************************
+
+Running ``provision -h`` shows all available options.
+
+.. code-block:: shell
+
+  (nvflare_venv) ~/workspace/repos/flare$ provision -h
+  usage: provision [-h] [-p PROJECT_FILE] [-o OUTPUT_DIR] [-n] [-u]
+
+  optional arguments:
+    -h, --help                                  show this help message and exit
+    -p PROJECT_FILE, --project_file PROJECT_FILE    file to describe FL project
+    -w WORKSPACE, --workspace WORKSPACE             directory used by provision
+    -u, --ui_tool         Run provisioning UI tool to generate project.yml file
+
+Running ``provision`` without any options and without a project.yml file in the current working directory will prompt
+to copy a default project.yml to the current working directory.
+
+*************************
+Provisioning tool UI page
+*************************
+
+The ``-u`` option will open the provisioning tool helper UI page in your browser.  This tool is built to help with
+setting up and generating a project.yml to work with the reference configuration of default bundled builders in the
+NVIDIA FLARE package. You may need to add to or edit the builders section before running ``provision -p project.yml`` if you
+have customized builders.
+
+.. image:: ../resources/provisioning_ui.png
+    :height: 400px
diff --git a/docs/user_guide/workspace.rst b/docs/user_guide/workspace.rst
new file mode 100644
index 0000000000..722cb192f2
--- /dev/null
+++ b/docs/user_guide/workspace.rst
@@ -0,0 +1,76 @@
+######################
+NVIDIA FLARE Workspace
+######################
+
+NVIDIA FLARE maintains a workspace for keeping the FL apps and execution results of different ``run_number``
+experiments. The following is the workspace folder structure when running NVIDIA FLARE for the server and clients.
+
+******
+Server
+******
+::
+
+    server/
+        run_1/
+            app_server/
+                ...
+                config_fed_server.json
+            fl_app.txt
+        run_2/
+            app_server/
+                ...
+                config/
+                    config_fed_server.json
+            fl_app.txt
+        startup/
+            fed_server.json
+            log.config
+            start.sh
+            sub_start.sh
+
+
+******
+Server
+******
+::
+
+    clientA/
+        run_1/
+            app_clientA/
+                ...
+                config_fed_client.json
+            fl_app.txt
+        run_2/
+            app_clientA/
+                ...
+                config/
+                    config_fed_client.json
+            fl_app.txt
+        startup/
+            fed_client.json
+            log.config
+            start.sh
+            sub_start.sh
+
+The FL application and run data from different ``run_number`` are kept in the separate ``run_N`` folders for each run
+number ``N``. Under the ``run_N`` folder, the ``app_server`` folder holds the FL application content of the server. The
+``app_clientName`` folder holds the FL application content of the client. The config_fed_server.json and
+config_fed_client.json may be in the root folder of the FL App, or in a sub-folder (for example: config) of the FL App.
+
+The fl_app.txt indicates which FL App current ``run_number`` is using. The ``deploy_app`` command will deploy the
+corresponding FL app into the current ``run_number``. If the ``app_server`` or ``app_clientName`` folder
+already exists, all the contents within that folder will be wiped out and deployed a new FL application. If you would
+like to keep some execution results within the same ``run_number``, you can keep them in the root directory of the
+``run_N`` folder.
+
+The ``startup`` folder contains the config and the scripts to start the FL server and client program.
+
+The :class:`Workspace<nvflare.apis.workspace.Workspace>` object is available through the FLContext. From the Workspace,
+you can access each folder location accordingly::
+
+    workspace = fl_ctx.get_prop(FLContextKey.WORKSPACE_OBJECT)
+
+
+.. literalinclude:: ../../nvflare/apis/workspace.py
+    :language: python
+    :lines: 36-
diff --git a/examples/hello-cyclic/config/config_fed_client.json b/examples/hello-cyclic/config/config_fed_client.json
new file mode 100755
index 0000000000..5544ef527f
--- /dev/null
+++ b/examples/hello-cyclic/config/config_fed_client.json
@@ -0,0 +1,19 @@
+{
+  "format_version": 2,
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "trainer.SimpleTrainer",
+        "args": {
+          "epochs_per_round": 2
+        }
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": []
+
+}
\ No newline at end of file
diff --git a/examples/hello-cyclic/config/config_fed_server.json b/examples/hello-cyclic/config/config_fed_server.json
new file mode 100755
index 0000000000..bbb3985e69
--- /dev/null
+++ b/examples/hello-cyclic/config/config_fed_server.json
@@ -0,0 +1,35 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "tf2_model_persistor.TF2ModelPersistor",
+      "args": {
+        "save_name": "tf2weights.pickle"
+      }
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    }
+  ],
+  "workflows": [
+    {
+      "id": "cyclic_ctl",
+      "path": "nvflare.app_common.workflows.cyclic_ctl.CyclicController",
+      "args": {
+        "num_rounds": 2,
+        "task_assignment_timeout": 8,
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "task_name": "train"
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/examples/hello-cyclic/custom/__init__.py b/examples/hello-cyclic/custom/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/examples/hello-cyclic/custom/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/examples/hello-cyclic/custom/tf2_model_persistor.py b/examples/hello-cyclic/custom/tf2_model_persistor.py
new file mode 100644
index 0000000000..8fb3453bbd
--- /dev/null
+++ b/examples/hello-cyclic/custom/tf2_model_persistor.py
@@ -0,0 +1,114 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pickle
+import json
+
+import tensorflow as tf
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from tf2_net import Net
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.abstract.model import make_model_learnable
+
+
+class TF2ModelPersistor(ModelPersistor):
+    def __init__(self, save_name="tf2_model.pkl"):
+        super().__init__()
+        self.save_name = save_name
+
+    def _initialize(self, fl_ctx: FLContext):
+        # get save path from FLContext
+        app_root = fl_ctx.get_prop(FLContextKey.APP_ROOT)
+        env = None
+        run_args = fl_ctx.get_prop(FLContextKey.ARGS)
+        if run_args:
+            env_config_file_name = os.path.join(app_root, run_args.env)
+            if os.path.exists(env_config_file_name):
+                try:
+                    with open(env_config_file_name) as file:
+                        env = json.load(file)
+                except:
+                    self.system_panic(
+                        reason="error opening env config file {}".format(env_config_file_name), fl_ctx=fl_ctx
+                    )
+                    return
+
+        if env is not None:
+            if env.get("APP_CKPT_DIR", None):
+                fl_ctx.set_prop(AppConstants.LOG_DIR, env["APP_CKPT_DIR"], private=True, sticky=True)
+            if env.get("APP_CKPT") is not None:
+                fl_ctx.set_prop(
+                    AppConstants.CKPT_PRELOAD_PATH,
+                    env["APP_CKPT"],
+                    private=True,
+                    sticky=True,
+                )
+
+        log_dir = fl_ctx.get_prop(AppConstants.LOG_DIR)
+        if log_dir:
+            self.log_dir = os.path.join(app_root, log_dir)
+        else:
+            self.log_dir = app_root
+        self._pkl_save_path = os.path.join(self.log_dir, self.save_name)
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
+
+        fl_ctx.sync_sticky()
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        """
+            initialize and load the Model.
+
+        Args:
+            fl_ctx: FLContext
+
+        Returns:
+            Model object
+        """
+
+        if os.path.exists(self._pkl_save_path):
+            self.logger.info(f"Loading server weights")
+            with open(self._pkl_save_path, "rb") as f:
+                model_learnable = pickle.load(f)
+        else:
+            self.logger.info(f"Initializing server model")
+            network = Net()
+            loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+            network.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
+            _ = network(tf.keras.Input(shape=(28, 28)))
+            var_dict = {network.get_layer(index=key).name: value for key, value in enumerate(network.get_weights())}
+            model_learnable = make_model_learnable(var_dict, dict())
+        return model_learnable
+
+    def handle_event(self, event: str, fl_ctx: FLContext):
+        if event == EventType.START_RUN:
+            self._initialize(fl_ctx)
+
+    def save_model(self, model_learnable: ModelLearnable, fl_ctx: FLContext):
+        """
+            persist the Model object
+
+        Args:
+            model: Model object
+            fl_ctx: FLContext
+        """
+        model_learnable_info = {k: str(type(v)) for k, v in model_learnable.items()}
+        self.logger.info(f"Saving aggregated server weights: \n {model_learnable_info}")
+        with open(self._pkl_save_path, "wb") as f:
+            pickle.dump(model_learnable, f)
diff --git a/examples/hello-cyclic/custom/tf2_net.py b/examples/hello-cyclic/custom/tf2_net.py
new file mode 100644
index 0000000000..526eeea0ca
--- /dev/null
+++ b/examples/hello-cyclic/custom/tf2_net.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tensorflow as tf
+
+
+class Net(tf.keras.Model):
+    def __init__(self):
+        super().__init__()
+        self.flatten = tf.keras.layers.Flatten(input_shape=(28, 28))
+        self.dense1 = tf.keras.layers.Dense(128, activation="relu")
+        self.dropout = tf.keras.layers.Dropout(0.2)
+        self.dense2 = tf.keras.layers.Dense(10)
+
+    def call(self, x):
+        x = self.flatten(x)
+        x = self.dense1(x)
+        x = self.dropout(x)
+        x = self.dense2(x)
+        return x
diff --git a/examples/hello-cyclic/custom/trainer.py b/examples/hello-cyclic/custom/trainer.py
new file mode 100644
index 0000000000..2355836207
--- /dev/null
+++ b/examples/hello-cyclic/custom/trainer.py
@@ -0,0 +1,142 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tensorflow as tf
+import numpy as np
+
+from nvflare.apis.dxo import DXO, DataKind, from_shareable
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.event_type import EventType
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.apis.signal import Signal
+
+from tf2_net import Net
+
+
+class SimpleTrainer(Executor):
+    def __init__(self, epochs_per_round):
+        super().__init__()
+        self.epochs_per_round = epochs_per_round
+        self.train_images, self.train_labels = None, None
+        self.test_images, self.test_labels = None, None
+        self.model = None
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.setup(fl_ctx)
+
+    def setup(self, fl_ctx: FLContext):
+        (self.train_images, self.train_labels), (
+            self.test_images,
+            self.test_labels,
+        ) = tf.keras.datasets.mnist.load_data()
+        self.train_images, self.test_images = (
+            self.train_images / 255.0,
+            self.test_images / 255.0,
+        )
+
+        # simulate separate datasets for each client by dividing MNIST dataset in half
+        client_name = fl_ctx.get_identity_name()
+        if client_name == "site-1":
+            self.train_images = self.train_images[: len(self.train_images) // 2]
+            self.train_labels = self.train_labels[: len(self.train_labels) // 2]
+            self.test_images = self.test_images[: len(self.test_images) // 2]
+            self.test_labels = self.test_labels[: len(self.test_labels) // 2]
+        elif client_name == "site-2":
+            self.train_images = self.train_images[len(self.train_images) // 2 :]
+            self.train_labels = self.train_labels[len(self.train_labels) // 2 :]
+            self.test_images = self.test_images[len(self.test_images) // 2 :]
+            self.test_labels = self.test_labels[len(self.test_labels) // 2 :]
+
+        model = Net()
+
+        loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+        model.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
+        _ = model(tf.keras.Input(shape=(28, 28)))
+        self.var_list = [
+            model.get_layer(index=index).name
+            for index in range(len(model.get_weights()))
+        ]
+        self.model = model
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        """
+        This function is an extended function from the super class.
+        As a supervised learning based trainer, the train function will run
+        evaluate and train engines based on model weights from `shareable`.
+        After finishing training, a new `Shareable` object will be submitted
+        to server for aggregation.
+
+        Args:
+            task_name: dispatched task
+            shareable: the `Shareable` object acheived from server.
+            fl_ctx: the `FLContext` object achieved from server.
+            abort_signal: if triggered, the training will be aborted.
+
+        Returns:
+            a new `Shareable` object to be submitted to server for aggregation.
+        """
+
+        # retrieve model weights download from server's shareable
+        if abort_signal.triggered:
+            return make_reply(ReturnCode.OK)
+
+        if task_name != "train":
+            return shareable
+
+        dxo = from_shareable(shareable)
+        model_weights = dxo.data
+
+        # use previous round's client weights to replace excluded layers from server
+        prev_weights = {
+            self.model.get_layer(index=key).name: value
+            for key, value in enumerate(self.model.get_weights())
+        }
+        print("dxo")
+        ordered_model_weights = {key: model_weights.get(key) for key in prev_weights}
+        for key in self.var_list:
+            value = ordered_model_weights.get(key)
+            if np.all(value == 0):
+                ordered_model_weights[key] = prev_weights[key]
+
+        # update local model weights with received weights
+        self.model.set_weights(list(ordered_model_weights.values()))
+
+        # adjust LR or other training time info as needed
+        # such as callback in the fit function
+        self.model.fit(
+            self.train_images,
+            self.train_labels,
+            epochs=self.epochs_per_round,
+            validation_data=(self.test_images, self.test_labels),
+        )
+
+        # report updated weights in shareable
+        weights = {
+            self.model.get_layer(index=key).name: value
+            for key, value in enumerate(self.model.get_weights())
+        }
+        dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights)
+
+        self.log_info(fl_ctx, "Local epochs finished. Returning shareable")
+
+        return dxo.update_shareable(shareable)
diff --git a/examples/hello-numpy-cross-val/config/config_fed_client.json b/examples/hello-numpy-cross-val/config/config_fed_client.json
new file mode 100755
index 0000000000..0812429b34
--- /dev/null
+++ b/examples/hello-numpy-cross-val/config/config_fed_client.json
@@ -0,0 +1,26 @@
+{
+  "format_version": 2,
+  "executors": [
+    {
+      "tasks": [
+        "train",
+        "submit_model"
+      ],
+      "executor": {
+        "path": "np_trainer.NPTrainer",
+        "args": {}
+      }
+    },
+    {
+      "tasks": [
+        "validate"
+      ],
+      "executor": {
+        "path": "np_validator.NPValidator"
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}
\ No newline at end of file
diff --git a/examples/hello-numpy-cross-val/config/config_fed_server.json b/examples/hello-numpy-cross-val/config/config_fed_server.json
new file mode 100755
index 0000000000..cc6e1cd6a9
--- /dev/null
+++ b/examples/hello-numpy-cross-val/config/config_fed_server.json
@@ -0,0 +1,64 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    },
+    {
+      "id": "model_locator",
+      "path": "np_model_locator.NPModelLocator",
+      "args": {}
+    },
+    {
+      "id": "formatter",
+      "path": "np_formatter.NPFormatter",
+      "args": {}
+    }
+  ],
+  "workflows": [
+    {
+      "id": "scatter_and_gather",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 6000
+      }
+    },
+    {
+      "id": "cross_site_model_eval",
+      "path": "nvflare.app_common.workflows.cross_site_model_eval.CrossSiteModelEval",
+      "args": {
+        "model_locator_id": "model_locator",
+        "submit_model_timeout": 600,
+        "validation_timeout": 6000,
+        "cleanup_models": false
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/examples/hello-numpy/custom/__init__.py b/examples/hello-numpy-cross-val/custom/__init__.py
similarity index 100%
rename from examples/hello-numpy/custom/__init__.py
rename to examples/hello-numpy-cross-val/custom/__init__.py
diff --git a/examples/hello-numpy-cross-val/custom/constants.py b/examples/hello-numpy-cross-val/custom/constants.py
new file mode 100755
index 0000000000..1f4784778a
--- /dev/null
+++ b/examples/hello-numpy-cross-val/custom/constants.py
@@ -0,0 +1,16 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class NPConstants:
+    NUMPY_KEY = "numpy_key"
diff --git a/examples/hello-numpy-cross-val/custom/np_formatter.py b/examples/hello-numpy-cross-val/custom/np_formatter.py
new file mode 100755
index 0000000000..59e93c4375
--- /dev/null
+++ b/examples/hello-numpy-cross-val/custom/np_formatter.py
@@ -0,0 +1,65 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.dxo import from_bytes, DataKind
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.formatter import \
+    Formatter
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPFormatter(Formatter):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def format(self, fl_ctx: FLContext) -> str:
+        """The format function gets validation shareable locations from the dictionary. It loads each shareable,
+        get the validation results and converts it into human readable string.
+
+        Args:
+            fl_ctx (FLContext): FLContext object.
+
+        Returns:
+            str: Human readable validation results.
+        """
+        # Get the val shareables
+        validation_shareables_dict = fl_ctx.get_prop(
+            AppConstants.VALIDATION_RESULT, {}
+        )
+
+        # Result dictionary
+        res = {}
+
+        try:
+            # This is a 2d dictionary with each validation result at
+            # validation_shareables_dict[data_client][model_client]
+            for data_client in validation_shareables_dict.keys():
+                validation_dict = validation_shareables_dict[data_client]
+                if validation_dict:
+                    res[data_client] = {}
+                    for model_name in validation_dict.keys():
+                        dxo_path = validation_dict[model_name]
+
+                        # Load the shareable
+                        with open(dxo_path, "rb") as f:
+                            metric_dxo = from_bytes(f.read())
+
+                        # Get metrics from shareable
+                        if metric_dxo and metric_dxo.data_kind== DataKind.METRICS:
+                            metrics = metric_dxo.data
+                            res[data_client][model_name] = metrics
+        except Exception as e:
+            self.log_error(fl_ctx, f"Exception: {e.__str__()}")
+
+        return f"{res}"
diff --git a/examples/hello-numpy-cross-val/custom/np_model_locator.py b/examples/hello-numpy-cross-val/custom/np_model_locator.py
new file mode 100755
index 0000000000..9e843a0afa
--- /dev/null
+++ b/examples/hello-numpy-cross-val/custom/np_model_locator.py
@@ -0,0 +1,80 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import List
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, DataKind
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model_locator import (
+    ModelLocator,
+)
+
+
+class NPModelLocator(ModelLocator):
+    SERVER_MODEL_NAME = "server"
+
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        """The ModelLocator's job is to find the models to be included for cross site evaluation
+        located on server. This NPModelLocator finds and extracts "server" model that is saved during training.
+
+        Args:
+            model_dir (str): Directory to look for models in. Defaults to "model"
+            model_name (str). Name of the model. Defaults to "server.npy"
+        """
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_file_name = model_name
+
+    def get_model_names(self, fl_ctx: FLContext) -> List[str]:
+        """Returns the list of model names that should be included from server in cross site validation.add()
+
+        Args:
+            fl_ctx (FLContext): FL Context object.
+
+        Returns:
+            List[str]: List of model names.
+        """
+        return [NPModelLocator.SERVER_MODEL_NAME]
+
+    def locate_model(self, model_name, fl_ctx: FLContext) -> DXO:
+        dxo = None
+        engine = fl_ctx.get_engine()
+
+        if model_name == NPModelLocator.SERVER_MODEL_NAME:
+            try:
+                run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+                run_dir = engine.get_workspace().get_run_dir(run_number)
+                model_path = os.path.join(run_dir, self.model_dir)
+
+                model_load_path = os.path.join(model_path, self.model_file_name)
+                np_data = None
+                try:
+                    np_data = np.load(model_load_path, allow_pickle=True)
+                    self.log_info(fl_ctx, f"Loaded {model_name} model from {model_load_path}.")
+                except Exception as e:
+                    self.log_error(fl_ctx, f"Unable to load NP Model: {e}.")
+
+                if np_data is not None:
+                    weights = {NPConstants.NUMPY_KEY: np_data}
+                    dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights, meta={})
+            except:
+                self.log_exception(fl_ctx, f"Exception in retrieving {NPModelLocator.SERVER_MODEL_NAME} model.")
+
+        return dxo
diff --git a/examples/hello-numpy-cross-val/custom/np_model_persistor.py b/examples/hello-numpy-cross-val/custom/np_model_persistor.py
new file mode 100755
index 0000000000..76fcc9c6c5
--- /dev/null
+++ b/examples/hello-numpy-cross-val/custom/np_model_persistor.py
@@ -0,0 +1,80 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable, make_model_learnable, ModelLearnableKey
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPModelPersistor(ModelPersistor):
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_name = model_name
+
+        # This is default model that will be used if not local model is provided.
+        self.default_data = np.array(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32
+        )
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        # Get start round from FLContext. If start_round > 0, we will try loading model from disk.
+        start_round = fl_ctx.get_prop(AppConstants.START_ROUND, 0)
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir, self.model_name)
+
+        # Create a new numpy model
+        if start_round > 0:
+            try:
+                data = np.load(model_path)
+            except Exception as e:
+                self.log_exception(
+                    fl_ctx,
+                    f"Unable to load model from {model_path}. Using default data instead.",
+                    fire_event=False
+                )
+                data = self.default_data.copy()
+        else:
+            data = self.default_data.copy()
+
+        # Generate model dictionary and create model_learnable.
+        weights = {NPConstants.NUMPY_KEY: data}
+        model_learnable = make_model_learnable(weights, {})
+
+        self.logger.info(f"Loaded initial model: {model_learnable[ModelLearnableKey.WEIGHTS]}")
+        return model_learnable
+
+    def save_model(self, model: ModelLearnable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self.model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
diff --git a/examples/hello-numpy-cross-val/custom/np_trainer.py b/examples/hello-numpy-cross-val/custom/np_trainer.py
new file mode 100755
index 0000000000..2ca54b9d0e
--- /dev/null
+++ b/examples/hello-numpy-cross-val/custom/np_trainer.py
@@ -0,0 +1,216 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, from_shareable, DataKind
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPTrainer(Executor):
+    def __init__(
+        self,
+        delta=1,
+        sleep_time=0,
+        train_task_name=AppConstants.TASK_TRAIN,
+        submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL,
+        model_name="best_numpy.npy",
+        model_dir="model",
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        if not (isinstance(delta, float) or isinstance(delta, int)):
+            raise TypeError("delta must be an instance of float or int.")
+
+        self._delta = delta
+        self._model_name = model_name
+        self._model_dir = model_dir
+        self._sleep_time = sleep_time
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     Create all major components here. This is a simple app that doesn't need any components.
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any long tasks should check abort_signal regularly. Otherwise abort client
+        # will not work.
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._train_task_name:
+            # First we extract DXO from the shareable.
+            try:
+                incoming_dxo = from_shareable(shareable)
+            except BaseException as e:
+                self.system_panic(f"Unable to convert shareable to model definition. Exception {e.__str__()}", fl_ctx)
+                return self._get_exception_shareable()
+
+            # Information about workflow is retrieved from the shareable header.
+            current_round = shareable.get_header(AppConstants.CURRENT_ROUND, None)
+            total_rounds = shareable.get_header(AppConstants.NUM_ROUNDS, None)
+
+            # Ensure that data is of type weights. Extract model data.
+            if incoming_dxo.data_kind != DataKind.WEIGHTS:
+                self.system_panic("Model dex should be of kind DataKind.WEIGHTS.", fl_ctx)
+                return self._get_exception_shareable()
+            np_data = incoming_dxo.data
+
+            # Display properties.
+            self.log_info(fl_ctx, f"Incoming data kind: {incoming_dxo.data_kind}")
+            self.log_info(fl_ctx, f"Model: \n{np_data}")
+            self.log_info(fl_ctx, f"Current Round: {current_round}")
+            self.log_info(fl_ctx, f"Total Rounds: {total_rounds}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Doing some dummy training.
+            if np_data:
+                if NPConstants.NUMPY_KEY in np_data:
+                    np_data[NPConstants.NUMPY_KEY] += self._delta
+                else:
+                    self.log_error(fl_ctx, "numpy_key not found in model.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+                    return shareable
+            else:
+                self.log_error(fl_ctx, "No model weights found in shareable.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # We check abort_signal regularly to make sure
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Save local numpy model
+            try:
+                self._save_local_model(fl_ctx, np_data)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Exception in saving local model: {e}.")
+
+            self.log_info(
+                fl_ctx,
+                f"Model after training: {np_data}",
+            )
+
+            # Checking abort signal again.
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Prepare a DXO for our updated model. Create shareable and return
+            outgoing_dxo = DXO(data_kind=incoming_dxo.data_kind, data=np_data, meta={})
+            return outgoing_dxo.to_shareable()
+        elif task_name == self._submit_model_task_name:
+            # Retrieve the local model saved during training.
+            np_data = None
+            try:
+                np_data = self._load_local_model(fl_ctx)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load model: {e}")
+
+            # Checking abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Create DXO and shareable from model data.
+            model_shareable = Shareable()
+            if np_data:
+                outgoing_dxo = DXO(data_kind=DataKind.WEIGHTS, data=np_data)
+                model_shareable = outgoing_dxo.to_shareable()
+            else:
+                # Set return code.
+                self.log_error(fl_ctx, f"local model not found.")
+                model_shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+
+            return model_shareable
+        else:
+            # If unknown task name, set RC accordingly.
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+            return shareable
+
+    def _load_local_model(self, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+
+        model_load_path = os.path.join(model_path, self._model_name)
+        np_data = None
+        try:
+            np_data = np.load(model_load_path)
+        except Exception as e:
+            self.log_error(fl_ctx, f"Unable to load local model: {e.__str__()}")
+            return None
+
+        model = ModelLearnable()
+        model[NPConstants.NUMPY_KEY] = np_data
+
+        return model
+
+    def _save_local_model(self, fl_ctx: FLContext, model: dict):
+        # Save local model
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self._model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
+
+    def _get_exception_shareable(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/examples/hello-numpy-cross-val/custom/np_validator.py b/examples/hello-numpy-cross-val/custom/np_validator.py
new file mode 100755
index 0000000000..4698353a85
--- /dev/null
+++ b/examples/hello-numpy-cross-val/custom/np_validator.py
@@ -0,0 +1,141 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import from_shareable, DataKind, DXO
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPValidator(Executor):
+    def __init__(
+        self,
+        epsilon=1,
+        sleep_time=0,
+        validate_task_name=AppConstants.TASK_VALIDATION,
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        self.logger = logging.getLogger("NPValidator")
+        self._random_epsilon = epsilon
+        self._sleep_time = sleep_time
+        self._validate_task_name = validate_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     Create all major components here. This is a simple app that doesn't need any components.
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any long tasks should check abort_signal regularly. Otherwise abort client
+        # will not work.
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._abort_execution()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._validate_task_name:
+            # First we extract DXO from the shareable.
+            try:
+                model_dxo = from_shareable(shareable)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to extract model dxo from shareable. Exception: {e.__str__()}")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # Get model from shareable. data_kind must be WEIGHTS.
+            if model_dxo.data and model_dxo.data_kind == DataKind.WEIGHTS:
+                model = model_dxo.data
+            else:
+                self.log_error(fl_ctx, f"Model DXO doesn't have data or is not of type DataKind.WEIGHTS. Unable  "
+                                       "to validate.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # The workflow provides MODEL_OWNER information in the shareable header.
+            model_name = shareable.get_header(AppConstants.MODEL_OWNER, "?")
+
+            # Print properties.
+            self.log_info(fl_ctx, f"Model: \n{model}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+            self.log_info(fl_ctx, f"Validating model from {model_name}.")
+
+            # Check abort signal regularly.
+            if abort_signal.triggered:
+                return self._abort_execution()
+
+            # Check if key exists in model
+            if NPConstants.NUMPY_KEY not in model:
+                self.log_error(fl_ctx, "numpy_key not in model. Unable to validate.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # Do some dummy validation.
+            random_epsilon = np.random.random()
+            self.log_info(
+                fl_ctx, f"Adding random epsilon {random_epsilon} in validation."
+            )
+            val_results = {}
+            np_data = model[NPConstants.NUMPY_KEY]
+            np_data = np.sum(np_data / np.max(np_data))
+            val_results["accuracy"] = np_data + random_epsilon
+
+            # Check abort signal regularly.
+            if abort_signal.triggered:
+                return self._abort_execution()
+
+            self.log_info(fl_ctx, f"Validation result: {val_results}")
+
+            # Create DXO for metrics and return shareable.
+            metric_dxo = DXO(data_kind=DataKind.METRICS, data=val_results)
+            return metric_dxo.to_shareable()
+
+        else:
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+            return shareable
+
+    def _abort_execution(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/examples/hello-numpy-sag/config/config_fed_client.json b/examples/hello-numpy-sag/config/config_fed_client.json
new file mode 100755
index 0000000000..33ed6495fe
--- /dev/null
+++ b/examples/hello-numpy-sag/config/config_fed_client.json
@@ -0,0 +1,17 @@
+{
+  "format_version": 2,
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "np_trainer.NPTrainer",
+        "args": {}
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}
\ No newline at end of file
diff --git a/examples/hello-numpy-sag/config/config_fed_server.json b/examples/hello-numpy-sag/config/config_fed_server.json
new file mode 100755
index 0000000000..188bf0089d
--- /dev/null
+++ b/examples/hello-numpy-sag/config/config_fed_server.json
@@ -0,0 +1,44 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    }
+  ],
+  "workflows": [
+    {
+      "id": "scatter_and_gather",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 6000
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/examples/hello-numpy-sag/custom/__init__.py b/examples/hello-numpy-sag/custom/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/examples/hello-numpy-sag/custom/constants.py b/examples/hello-numpy-sag/custom/constants.py
new file mode 100755
index 0000000000..1f4784778a
--- /dev/null
+++ b/examples/hello-numpy-sag/custom/constants.py
@@ -0,0 +1,16 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class NPConstants:
+    NUMPY_KEY = "numpy_key"
diff --git a/examples/hello-numpy-sag/custom/np_model_persistor.py b/examples/hello-numpy-sag/custom/np_model_persistor.py
new file mode 100755
index 0000000000..76fcc9c6c5
--- /dev/null
+++ b/examples/hello-numpy-sag/custom/np_model_persistor.py
@@ -0,0 +1,80 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable, make_model_learnable, ModelLearnableKey
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPModelPersistor(ModelPersistor):
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_name = model_name
+
+        # This is default model that will be used if not local model is provided.
+        self.default_data = np.array(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32
+        )
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        # Get start round from FLContext. If start_round > 0, we will try loading model from disk.
+        start_round = fl_ctx.get_prop(AppConstants.START_ROUND, 0)
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir, self.model_name)
+
+        # Create a new numpy model
+        if start_round > 0:
+            try:
+                data = np.load(model_path)
+            except Exception as e:
+                self.log_exception(
+                    fl_ctx,
+                    f"Unable to load model from {model_path}. Using default data instead.",
+                    fire_event=False
+                )
+                data = self.default_data.copy()
+        else:
+            data = self.default_data.copy()
+
+        # Generate model dictionary and create model_learnable.
+        weights = {NPConstants.NUMPY_KEY: data}
+        model_learnable = make_model_learnable(weights, {})
+
+        self.logger.info(f"Loaded initial model: {model_learnable[ModelLearnableKey.WEIGHTS]}")
+        return model_learnable
+
+    def save_model(self, model: ModelLearnable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self.model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
diff --git a/examples/hello-numpy-sag/custom/np_trainer.py b/examples/hello-numpy-sag/custom/np_trainer.py
new file mode 100755
index 0000000000..d65b609acc
--- /dev/null
+++ b/examples/hello-numpy-sag/custom/np_trainer.py
@@ -0,0 +1,192 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, from_shareable, DataKind
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPTrainer(Executor):
+    def __init__(
+        self,
+        delta=1,
+        sleep_time=0,
+        train_task_name=AppConstants.TASK_TRAIN,
+        submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL,
+        model_name="best_numpy.npy",
+        model_dir="model",
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        if not (isinstance(delta, float) or isinstance(delta, int)):
+            raise TypeError("delta must be an instance of float or int.")
+
+        self._delta = delta
+        self._model_name = model_name
+        self._model_dir = model_dir
+        self._sleep_time = sleep_time
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     Create all major components here. This is a simple app that doesn't need any components.
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any long tasks should check abort_signal regularly. Otherwise abort client
+        # will not work.
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._train_task_name:
+            # First we extract DXO from the shareable.
+            try:
+                incoming_dxo = from_shareable(shareable)
+            except BaseException as e:
+                self.system_panic(f"Unable to convert shareable to model definition. Exception {e.__str__()}", fl_ctx)
+                return self._get_exception_shareable()
+
+            # Information about workflow is retrieved from the shareable header.
+            current_round = shareable.get_header(AppConstants.CURRENT_ROUND, None)
+            total_rounds = shareable.get_header(AppConstants.NUM_ROUNDS, None)
+
+            # Ensure that data is of type weights. Extract model data.
+            if incoming_dxo.data_kind != DataKind.WEIGHTS:
+                self.system_panic("Model dex should be of kind DataKind.WEIGHTS.", fl_ctx)
+                return self._get_exception_shareable()
+            np_data = incoming_dxo.data
+
+            # Display properties.
+            self.log_info(fl_ctx, f"Incoming data kind: {incoming_dxo.data_kind}")
+            self.log_info(fl_ctx, f"Model: \n{np_data}")
+            self.log_info(fl_ctx, f"Current Round: {current_round}")
+            self.log_info(fl_ctx, f"Total Rounds: {total_rounds}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Doing some dummy training.
+            if np_data:
+                if NPConstants.NUMPY_KEY in np_data:
+                    np_data[NPConstants.NUMPY_KEY] += self._delta
+                else:
+                    self.log_error(fl_ctx, "numpy_key not found in model.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+                    return shareable
+            else:
+                self.log_error(fl_ctx, "No model weights found in shareable.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # We check abort_signal regularly to make sure
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Save local numpy model
+            try:
+                self._save_local_model(fl_ctx, np_data)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Exception in saving local model: {e}.")
+
+            self.log_info(
+                fl_ctx,
+                f"Model after training: {np_data}",
+            )
+
+            # Checking abort signal again.
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Prepare a DXO for our updated model. Create shareable and return
+            outgoing_dxo = DXO(data_kind=incoming_dxo.data_kind, data=np_data, meta={})
+            return outgoing_dxo.to_shareable()
+        else:
+            # If unknown task name, set RC accordingly.
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+            return shareable
+
+    def _load_local_model(self, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+
+        model_load_path = os.path.join(model_path, self._model_name)
+        try:
+            np_data = np.load(model_load_path)
+        except Exception as e:
+            self.log_error(fl_ctx, f"Unable to load local model: {e.__str__()}")
+            return None
+
+        model = ModelLearnable()
+        model[NPConstants.NUMPY_KEY] = np_data
+
+        return model
+
+    def _save_local_model(self, fl_ctx: FLContext, model: dict):
+        # Save local model
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self._model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
+
+    def _get_exception_shareable(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/examples/hello-numpy/config/config_fed_client.json b/examples/hello-numpy/config/config_fed_client.json
deleted file mode 100644
index b22e52f217..0000000000
--- a/examples/hello-numpy/config/config_fed_client.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
-  "format_version": 1,
-  "platform": "numpy",
-  "client": {
-    "outbound_filters": [],
-    "inbound_filters": []
-  },
-  "client_trainer": {
-    "path": "trainer.SimpleTrainer",
-    "args": {}
-  },
-  "handlers": []
-}
diff --git a/examples/hello-numpy/config/config_fed_server.json b/examples/hello-numpy/config/config_fed_server.json
deleted file mode 100644
index 1cde0ac65a..0000000000
--- a/examples/hello-numpy/config/config_fed_server.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
-  "format_version": 1,
-  "servers": [
-    {
-      "min_num_clients": 1,
-      "max_num_clients": 100,
-      "wait_after_min_clients": 10,
-      "heart_beat_timeout": 600,
-      "start_round": 0,
-      "num_rounds": 10
-    }
-  ],
-  "aggregator": {
-    "name": "AccumulateWeightedAggregator",
-    "args": {
-      "aggregation_weights":
-        {
-          "site-1": 1,
-          "site-2": 1
-        }
-    }
-  },
-  "outbound_filters": [],
-  "inbound_filters": [],
-  "persistor": {
-    "path": "model_persistor.NumpyModelPersistor",
-    "args": {
-      "save_name": "weight.pickle"
-    }
-  },
-  "shareable_generator": {
-    "name": "FullModelShareableGenerator"
-  }
-}
diff --git a/examples/hello-numpy/custom/model_persistor.py b/examples/hello-numpy/custom/model_persistor.py
deleted file mode 100644
index e20c58f891..0000000000
--- a/examples/hello-numpy/custom/model_persistor.py
+++ /dev/null
@@ -1,59 +0,0 @@
-import os
-import pickle
-import numpy as np
-
-from nvflare.apis import Model, ModelPersistor
-from nvflare.apis.fl_context import FLContext
-from nvflare.apis.fl_constant import FLConstants
-
-
-class NumpyModelPersistor(ModelPersistor):
-    def __init__(self, save_name=None):
-        super().__init__()
-        self.save_name = save_name
-
-    def _initialize(self, fl_ctx: FLContext):
-        # get save path from FLContext
-        train_root = fl_ctx.get_prop(FLConstants.TRAIN_ROOT)
-        log_dir = fl_ctx.get_prop(FLConstants.LOG_DIR)
-        if log_dir:
-            self.log_dir = os.path.join(train_root, log_dir)
-        else:
-            self.log_dir = train_root
-        self._pkl_save_path = os.path.join(self.log_dir, self.save_name)
-        if not os.path.exists(self.log_dir):
-            os.makedirs(self.log_dir)
-
-    def load_model(self, fl_ctx: FLContext) -> Model:
-        """
-            initialize and load the Model.
-
-        Args:
-            fl_ctx: FLContext
-
-        Returns:
-            Model object
-        """
-        self._initialize(fl_ctx)
-
-        model = Model()
-        if os.path.exists(self._pkl_save_path):
-            with open(self._pkl_save_path, "rb") as f:
-                d = pickle.load(f)
-                self.logger.info(f"Loading server weights: {d} \n")
-            model.update(d)
-        else:
-            model = {"sequence": np.array([0, 1])}
-        return model
-
-    def save_model(self, model: Model, fl_ctx: FLContext):
-        """
-            persist the Model object
-
-        Args:
-            model: Model object
-            fl_ctx: FLContext
-        """
-        self.logger.info(f"Saving aggregated server weights: {model} \n")
-        with open(self._pkl_save_path, "wb") as f:
-            pickle.dump(model, f)
diff --git a/examples/hello-numpy/custom/trainer.py b/examples/hello-numpy/custom/trainer.py
deleted file mode 100644
index ea4ca9078d..0000000000
--- a/examples/hello-numpy/custom/trainer.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import numpy as np
-
-from nvflare.apis.fl_constant import FLConstants
-from nvflare.apis.fl_context import FLContext
-from nvflare.apis.shareable import Shareable, ShareableKey, ShareableValue
-from nvflare.apis.trainer import Trainer
-
-class SimpleTrainer(Trainer):
-
-    def train(self, shareable: Shareable, fl_ctx: FLContext, abort_signal) -> Shareable:
-        """
-        This function is an extended function from the super class.
-        As a supervised learning based trainer, the train function will run
-        evaluate and train engines based on model weights from `shareable`.
-        After the training is done, a new `Shareable` object will be submitted
-        to server for aggregation.
-
-        Args:
-            shareable: the `Shareable` object received from server.
-            fl_ctx: the `FLContext` object achieved from server.
-            abort_signal: if triggered, the training will be aborted.
-
-        Returns:
-            a new `Shareable` object to be submitted to server for aggregation.
-        """
-
-        # retrieve model weights download from server's shareable
-        fib = shareable[ShareableKey.MODEL_WEIGHTS]
-        terms = fib["sequence"]
-
-        # perform one step of Fibonacci 
-        new_fib = {"sequence": np.array([terms[1], terms[0] + terms[1]])}
-
-        # add -1 and +1 bias for site-1 and site-2 respectively
-        client_name = fl_ctx.get_prop(FLConstants.CLIENT_NAME)
-        if client_name == "site-1":
-            new_fib["sequence"] -= 1
-        elif client_name == "site-2":
-            new_fib["sequence"] += 1
-
-        self.logger.info(f"{client_name} Shareable from server: {fib}")
-        self.logger.info(f"{client_name} New Shareable to server: {new_fib} \n")
-
-        # send updated weights back to server through new shareable
-        shareable = Shareable()
-        shareable[ShareableKey.META] = {FLConstants.NUM_STEPS_CURRENT_ROUND: 1}
-        shareable[ShareableKey.TYPE] = ShareableValue.TYPE_WEIGHTS
-        shareable[ShareableKey.DATA_TYPE] = ShareableValue.DATA_TYPE_UNENCRYPTED
-        shareable[ShareableKey.MODEL_WEIGHTS] = new_fib
-
-        return shareable
diff --git a/examples/hello-pt/config/config_fed_client.json b/examples/hello-pt/config/config_fed_client.json
index 2524442956..4e8dae513d 100644
--- a/examples/hello-pt/config/config_fed_client.json
+++ b/examples/hello-pt/config/config_fed_client.json
@@ -1,13 +1,30 @@
 {
-  "format_version": 1,
-  "platform": "PT",
-  "client": {
-    "outbound_filters": [],
-    "inbound_filters": []
-  },
-  "client_trainer": {
-    "path": "trainer.SimpleTrainer",
-    "args": {}
-  },
-  "handlers": []
-}
\ No newline at end of file
+  "format_version": 2,
+
+  "executors": [
+    {
+      "tasks": ["train", "submit_model"],
+      "executor": {
+        "path": "cifar10trainer.Cifar10Trainer",
+        "args": {
+          "lr": 0.01,
+          "epochs": 1
+        }
+      }
+    },
+    {
+      "tasks": ["validate"],
+      "executor": {
+        "path": "cifar10validator.Cifar10Validator",
+        "args": {
+        }
+      }
+    }
+  ],
+  "task_result_filters": [
+  ],
+  "task_data_filters": [
+  ],
+  "components": [
+  ]
+}
diff --git a/examples/hello-pt/config/config_fed_server.json b/examples/hello-pt/config/config_fed_server.json
index 364d5b34e3..3775fafd03 100644
--- a/examples/hello-pt/config/config_fed_server.json
+++ b/examples/hello-pt/config/config_fed_server.json
@@ -1,36 +1,69 @@
 {
-  "format_version": 1,
-  "servers": [
+  "format_version": 2,
+
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "name": "PTFileModelPersistor",
+      "args": {
+        "model": {
+          "path": "net.SimpleNetwork"
+        }
+      }
+    },
     {
-      "min_num_clients": 1,
-      "max_num_clients": 100,
-      "wait_after_min_clients": 10,
-      "heart_beat_timeout": 600,
-      "start_round": 0,
-      "num_rounds": 3
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    },
+    {
+      "id": "model_locator",
+      "path": "pt_model_locator.PTModelLocator",
+      "args": {
+
+      }
+    },
+    {
+      "id": "json_generator",
+      "path": "validation_json_generator.ValidationJsonGenerator",
+      "args": {
+      }
     }
   ],
-  "aggregator": {
-    "name": "AccumulateWeightedAggregator",
-    "args": {
-      "aggregation_weights":
-        {
-          "site-1": 1,
-          "site-2": 1
+  "workflows": [
+      {
+        "id": "scatter_and_gather",
+        "name": "ScatterAndGather",
+        "args": {
+            "min_clients" : 2,
+            "num_rounds" : 2,
+            "start_round": 0,
+            "wait_time_after_min_received": 10,
+            "aggregator_id": "aggregator",
+            "persistor_id": "persistor",
+            "shareable_generator_id": "shareable_generator",
+            "train_task_name": "train",
+            "train_timeout": 0
+        }
+      },
+      {
+        "id": "cross_site_validate",
+        "name": "CrossSiteModelEval",
+        "args": {
+          "model_locator_id": "model_locator"
         }
-    }
-  },
-  "outbound_filters": [],
-  "inbound_filters": [],
-  "persistor": {
-    "name": "PTFileModelPersistor",
-    "args": {
-      "model": {
-        "path": "net.Net"
       }
-    }
-  },
-  "shareable_generator": {
-    "name": "FullModelShareableGenerator"
-  }
-}
\ No newline at end of file
+  ]
+}
diff --git a/examples/hello-pt/custom/cifar10trainer.py b/examples/hello-pt/custom/cifar10trainer.py
new file mode 100644
index 0000000000..b25d94a0ee
--- /dev/null
+++ b/examples/hello-pt/custom/cifar10trainer.py
@@ -0,0 +1,163 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+
+import torch
+import torchvision
+from torch import nn
+from torchvision import transforms
+
+from net import SimpleNetwork
+from nvflare.apis.dxo import from_shareable, DXO, DataKind, MetaKey
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode, ReservedKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.model import make_model_learnable, model_learnable_to_dxo
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.pt.pt_fed_utils import PTModelPersistenceFormatManager
+from pt_constants import PTConstants
+
+
+class Cifar10Trainer(Executor):
+
+    def __init__(self, lr=0.01, epochs=5, train_task_name=AppConstants.TASK_TRAIN,
+                 submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL, exclude_vars=None):
+        super(Cifar10Trainer, self).__init__()
+
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+        # Training setup
+        self.model = SimpleNetwork()
+        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+        self.model.to(self.device)
+        self.loss = nn.CrossEntropyLoss()
+        self.optimizer = torch.optim.SGD(self.model.parameters(), lr=lr, momentum=0.9)
+        self.lr = lr
+        self.transforms = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
+        ])
+        self.epochs = epochs
+        self.exclude_vars = exclude_vars
+
+        # Cifar10 dataset for training.
+        self.train_dataset = torchvision.datasets.CIFAR10(root='~/data', transform=self.transforms,
+                                                          download=True, train=True)
+        self.train_loader = torch.utils.data.DataLoader(
+            self.train_dataset, batch_size=4, shuffle=True
+        )
+        self.n_iterations = len(self.train_loader)
+
+        # # Setup the persistence manager to save PT model.
+        self.default_train_conf = {"train": {"model": type(self.model).__name__}}
+        self.persistence_manager = PTModelPersistenceFormatManager(data=self.model.state_dict(),
+                                                                   default_train_conf=self.default_train_conf)
+
+    def local_train(self, fl_ctx, weights):
+        # Set the model weights
+        self.model.load_state_dict(state_dict=weights)
+
+        # Basic training
+        self.model.train()
+        for epoch in range(self.epochs):
+            running_loss = 0
+            for i, batch in enumerate(self.train_loader):
+                images, labels = batch[0].to(self.device), batch[1].to(self.device)
+                self.optimizer.zero_grad()
+
+                predictions = self.model(images)
+                cost = self.loss(predictions, labels)
+                cost.backward()
+                self.optimizer.step()
+
+                running_loss += (cost.cpu().detach().numpy()/images.size()[0])
+                if i % 3000 == 0:
+                    self.log_info(fl_ctx, f"Epoch: {epoch}/{self.epochs}, Iteration: {i}, "
+                                          f"Loss: {running_loss/3000}")
+                    running_loss = 0.0
+
+    def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable:
+        try:
+            if task_name == self._train_task_name:
+                # Get model weights
+                dxo = from_shareable(shareable)
+
+                # Check if dxo is valid.
+                if not isinstance(dxo, DXO):
+                    self.log_exception(fl_ctx, f"dxo excepted type DXO. Got {type(dxo)} instead.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                    return shareable
+
+                # Ensure data kind is weights.
+                if not dxo.data_kind == DataKind.WEIGHTS:
+                    self.log_exception(fl_ctx, f"data_kind expected WEIGHTS but got {dxo.data_kind} instead.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                    return shareable
+
+                # Convert weights to tensor. Run training
+                torch_weights = {k: torch.as_tensor(v) for k, v in dxo.data.items()}
+                self.local_train(fl_ctx, torch_weights)
+
+                self.save_local_model(fl_ctx)
+
+                # Get the new state dict and send as weights
+                new_weights = self.model.state_dict()
+                new_weights = {k: v.cpu().numpy() for k, v in new_weights.items()}
+
+                outgoing_dxo = DXO(data_kind=DataKind.WEIGHTS, data=new_weights,
+                                   meta={MetaKey.NUM_STEPS_CURRENT_ROUND: self.n_iterations})
+                return outgoing_dxo.to_shareable()
+            elif task_name == self._submit_model_task_name:
+                # Load local model
+                ml = self.load_local_model(fl_ctx)
+
+                # Get the model parameters and create dxo from it
+                dxo = model_learnable_to_dxo(ml)
+                return dxo.to_shareable()
+            else:
+                shareable = Shareable()
+                shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+                return shareable
+        except:
+            self.log_exception(fl_ctx, f"Exception in simple trainer.")
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+            return shareable
+
+    def save_local_model(self, fl_ctx: FLContext):
+        run_dir = fl_ctx.get_engine().get_workspace().get_run_dir(fl_ctx.get_prop(ReservedKey.RUN_NUM))
+        models_dir = os.path.join(run_dir, 'models')
+        if not os.path.exists(models_dir):
+            os.makedirs(models_dir)
+        model_path = os.path.join(models_dir, PTConstants.PTLocalModelName)
+
+        ml = make_model_learnable(self.model.state_dict(), {})
+        self.persistence_manager.update(ml)
+        torch.save(self.persistence_manager.to_persistence_dict(), model_path)
+
+    def load_local_model(self, fl_ctx: FLContext):
+        run_dir = fl_ctx.get_engine().get_workspace().get_run_dir(fl_ctx.get_prop(ReservedKey.RUN_NUM))
+        models_dir = os.path.join(run_dir, 'models')
+        if not os.path.exists(models_dir):
+            return None
+        model_path = os.path.join(models_dir, PTConstants.PTLocalModelName)
+
+        self.persistence_manager = PTModelPersistenceFormatManager(data=torch.load(model_path),
+                                                                   default_train_conf=self.default_train_conf)
+        ml = self.persistence_manager.to_model_learnable(exclude_vars=self.exclude_vars)
+        return ml
diff --git a/examples/hello-pt/custom/cifar10validator.py b/examples/hello-pt/custom/cifar10validator.py
new file mode 100644
index 0000000000..195633184c
--- /dev/null
+++ b/examples/hello-pt/custom/cifar10validator.py
@@ -0,0 +1,106 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import torchvision.datasets
+from torchvision import transforms
+
+from net import SimpleNetwork
+from nvflare.apis.dxo import from_shareable, DataKind, DXO
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.app_constant import AppConstants
+
+
+class Cifar10Validator(Executor):
+    
+    def __init__(self, validate_task_name=AppConstants.TASK_VALIDATION):
+        super(Cifar10Validator, self).__init__()
+
+        self._validate_task_name = validate_task_name
+
+        # Setup the model
+        self.model = SimpleNetwork()
+        self.transforms = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
+        ])
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.model.to(self.device)
+
+        # Preparing the dataset for testing.
+        self.test_data = torchvision.datasets.CIFAR10(root='~/data', train=False, transform=self.transforms)
+        self.test_loader = torch.utils.data.DataLoader(self.test_data, batch_size=4, shuffle=False)
+
+    def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable:
+        out_shareable = Shareable()
+        if task_name == self._validate_task_name:
+            try:
+                dxo = from_shareable(shareable)
+
+                # Check if dxo is valid.
+                if not dxo:
+                    self.log_exception(fl_ctx, "DXO invalid")
+                    out_shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                    return out_shareable
+
+                # Ensure data_kind is weights.
+                if not dxo.data_kind == DataKind.WEIGHTS:
+                    self.log_exception(fl_ctx, f"DXO is of type {dxo.data_kind} but expected type WEIGHTS.")
+                    out_shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                    return out_shareable
+
+                # Extract weights and ensure they are tensor.
+                model_owner = shareable.get_header(AppConstants.MODEL_OWNER, "?")
+                weights = dxo.data
+                weights = {k: torch.as_tensor(v, device=self.device) for k, v in weights.items()}
+
+                # Get validation accuracy
+                val_accuracy = self.do_validation(weights)
+                self.log_info(fl_ctx, f"Accuracy when validating {model_owner}'s model on"
+                                      f" {fl_ctx.get_identity_name()}"f's data: {val_accuracy}')
+
+                dxo = DXO(data_kind=DataKind.METRICS, data={'val_acc': val_accuracy})
+                return dxo.to_shareable()
+            except:
+                self.log_exception(fl_ctx, f"Exception in validating model from {model_owner}")
+                out_shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return out_shareable
+        else:
+            out_shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+            return out_shareable
+
+    def do_validation(self, weights):
+        self.model.load_state_dict(weights)
+
+        self.model.eval()
+
+        correct = 0
+        total = 0
+        with torch.no_grad():
+            for i, (images, labels) in enumerate(self.test_loader):
+                images, labels = images.to(self.device), labels.to(self.device)
+                output = self.model(images)
+
+                _, pred_label = torch.max(output, 1)
+
+                correct += (pred_label == labels).sum().item()
+                total += images.size()[0]
+
+            metric = correct/float(total)
+
+        return metric
diff --git a/examples/hello-pt/custom/net.py b/examples/hello-pt/custom/net.py
index f45dd5527d..0bd74c877d 100644
--- a/examples/hello-pt/custom/net.py
+++ b/examples/hello-pt/custom/net.py
@@ -1,13 +1,26 @@
-from __future__ import print_function
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
 
 
-class Net(nn.Module):
+class SimpleNetwork(nn.Module):
     def __init__(self):
-        super().__init__()
+        super(SimpleNetwork, self).__init__()
+
         self.conv1 = nn.Conv2d(3, 6, 5)
         self.pool = nn.MaxPool2d(2, 2)
         self.conv2 = nn.Conv2d(6, 16, 5)
diff --git a/examples/hello-pt/custom/pt_constants.py b/examples/hello-pt/custom/pt_constants.py
new file mode 100644
index 0000000000..52d5e2270c
--- /dev/null
+++ b/examples/hello-pt/custom/pt_constants.py
@@ -0,0 +1,18 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class PTConstants:
+    PTServerName = "server"
+    PTFileModelName = "FL_global_model.pt"
+    PTLocalModelName = "local_model.pt"
diff --git a/examples/hello-pt/custom/pt_model_locator.py b/examples/hello-pt/custom/pt_model_locator.py
new file mode 100644
index 0000000000..f5c7bfe700
--- /dev/null
+++ b/examples/hello-pt/custom/pt_model_locator.py
@@ -0,0 +1,65 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import List
+
+import torch.cuda
+
+from net import SimpleNetwork
+from nvflare.apis.dxo import DXO
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import model_learnable_to_dxo
+from nvflare.app_common.abstract.model_locator import ModelLocator
+from nvflare.app_common.pt.pt_fed_utils import PTModelPersistenceFormatManager
+from pt_constants import PTConstants
+
+
+class PTModelLocator(ModelLocator):
+
+    def __init__(self, exclude_vars=None, model=None):
+        super(PTModelLocator, self).__init__()
+
+        self.model = SimpleNetwork()
+        self.exclude_vars = exclude_vars
+
+    def get_model_names(self, fl_ctx: FLContext) -> List[str]:
+        return [PTConstants.PTServerName]
+
+    def locate_model(self, model_name, fl_ctx: FLContext) -> DXO:
+        if model_name == PTConstants.PTServerName:
+            server_run_dir = fl_ctx.get_engine().get_workspace().get_app_dir(fl_ctx.get_run_number())
+            model_path = os.path.join(server_run_dir, PTConstants.PTFileModelName)
+            if not os.path.exists(model_path):
+                return None
+
+            # Load the torch model
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+            data = torch.load(model_path, map_location=device)
+
+            # Setup the persistence manager.
+            if self.model:
+                default_train_conf = {"train": {"model": type(self.model).__name__}}
+            else:
+                default_train_conf = None
+
+            # Use persistence manager to get learnable
+            persistence_manager = PTModelPersistenceFormatManager(data, default_train_conf=default_train_conf)
+            ml = persistence_manager.to_model_learnable(exclude_vars=None)
+
+            # Create dxo and return
+            return model_learnable_to_dxo(ml)
+        else:
+            self.log_exception(fl_ctx, f"PTModelLocator doesn't recognize name: {model_name}")
+            return None
diff --git a/examples/hello-pt/custom/trainer.py b/examples/hello-pt/custom/trainer.py
deleted file mode 100644
index 1a7ad4ccec..0000000000
--- a/examples/hello-pt/custom/trainer.py
+++ /dev/null
@@ -1,87 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.optim as optim
-from net import Net
-from nvflare.apis import Trainer
-from nvflare.apis.fl_constant import FLConstants, ShareableKey, ShareableValue
-from nvflare.apis.fl_context import FLContext
-from nvflare.apis.shareable import Shareable
-from torchvision import datasets, transforms
-
-
-class SimpleTrainer(Trainer):
-    def __init__(self):
-        super().__init__()
-        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-        self.model = Net()
-        self.model.to(self.device)
-        self.optimizer = optim.SGD(self.model.parameters(), lr=0.001, momentum=0.9)
-        self.criterion = nn.CrossEntropyLoss()
-        self.transform = transforms.Compose(
-            [
-                transforms.ToTensor(),
-                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
-            ]
-        )
-        self.trainset = datasets.CIFAR10(
-            root="./data", train=True, download=True, transform=self.transform
-        )
-        self.train_loader = torch.utils.data.DataLoader(
-            self.trainset, batch_size=4, shuffle=True, num_workers=2
-        )
-
-    def local_train(self):
-        """A regular training procedure.
-
-        :return:
-        """
-        self.model.train()
-        for epoch in range(2):
-            # set the model to train mode
-            for i, (inputs, labels) in enumerate(self.train_loader):
-                inputs, labels = inputs.to(self.device), labels.to(self.device)
-                # zero the parameter gradients
-                self.optimizer.zero_grad()
-                # forward + backward + optimize
-                outputs = self.model(inputs)
-                loss = self.criterion(outputs, labels)
-                loss.backward()
-                self.optimizer.step()
-
-    def train(self, shareable: Shareable, fl_ctx: FLContext, abort_signal) -> Shareable:
-        """
-        This function is an extended function from the super class.
-        As a supervised learning based trainer, the train function will run
-        evaluate and train engines based on model weights from `shareable`.
-        After finishing training, a new `Shareable` object will be submitted
-        to server for aggregation.
-
-        Args:
-            shareable: the `Shareable` object acheived from server.
-            fl_ctx: the `FLContext` object achieved from server.
-            abort_signal: if triggered, the training will be aborted.
-
-        Returns:
-            a new `Shareable` object to be submitted to server for aggregation.
-        """
-        # retrieve model weights download from server's shareable
-        model_weights = shareable[ShareableKey.MODEL_WEIGHTS]
-
-        # update local model weights with received weights
-        self.model.load_state_dict(
-            {k: torch.as_tensor(v) for k, v in model_weights.items()}
-        )
-
-        self.local_train()
-
-        # build the shareable
-        shareable = Shareable()
-        shareable[ShareableKey.META] = {FLConstants.NUM_STEPS_CURRENT_ROUND: 1}
-        shareable[ShareableKey.TYPE] = ShareableValue.TYPE_WEIGHTS
-        shareable[ShareableKey.DATA_TYPE] = ShareableValue.DATA_TYPE_UNENCRYPTED
-        shareable[ShareableKey.MODEL_WEIGHTS] = {
-            k: v.cpu().numpy() for k, v in self.model.state_dict().items()
-        }
-
-        self.logger.info("Local epochs finished.  Returning shareable")
-        return shareable
diff --git a/examples/hello-pt/custom/validation_json_generator.py b/examples/hello-pt/custom/validation_json_generator.py
new file mode 100644
index 0000000000..f24eee49af
--- /dev/null
+++ b/examples/hello-pt/custom/validation_json_generator.py
@@ -0,0 +1,71 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os.path
+
+from nvflare.apis.dxo import from_shareable, DataKind
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.app_event_type import AppEventType
+
+
+class ValidationJsonGenerator(FLComponent):
+
+    def __init__(self, results_dir=AppConstants.CROSS_VAL_DIR, json_file_name="cross_val.json"):
+        super(ValidationJsonGenerator, self).__init__()
+
+        self.results_dir = results_dir
+        self.val_results = {}
+        self.json_file_name = json_file_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.val_results.clear()
+        elif event_type == AppEventType.VALIDATION_RESULT_RECEIVED:
+            model_owner = fl_ctx.get_prop(AppConstants.MODEL_OWNER, None)
+            data_client = fl_ctx.get_prop(AppConstants.DATA_CLIENT, None)
+            val_results = fl_ctx.get_prop(AppConstants.VALIDATION_RESULT, None)
+
+            if not model_owner:
+                self.log_error(fl_ctx, "model_owner unknown. Validation result will not be saved to json")
+            if not data_client:
+                self.log_error(fl_ctx, "data_client unknown. Validation result will not be saved to json")
+
+            if val_results:
+                try:
+                    dxo = from_shareable(val_results)
+                    dxo.validate()
+
+                    if dxo.data_kind == DataKind.METRICS:
+                        if data_client not in self.val_results:
+                            self.val_results[data_client] = {}
+                        self.val_results[data_client][model_owner] = dxo.data
+                    else:
+                        self.log_error(fl_ctx, f"Expected dxo of kind METRICS but got {dxo.data_kind} instead.")
+                except:
+                    self.log_exception(fl_ctx, f"Exception in handling validation result.")
+            else:
+                self.log_error(fl_ctx, "Validation result not found.")
+        elif event_type == EventType.END_RUN:
+            run_dir = fl_ctx.get_engine().get_workspace().get_run_dir(fl_ctx.get_run_number())
+            cross_val_res_dir = os.path.join(run_dir, self.results_dir)
+            if not os.path.exists(cross_val_res_dir):
+                os.makedirs(cross_val_res_dir)
+
+            res_file_path = os.path.join(cross_val_res_dir, self.json_file_name)
+            with open(res_file_path, 'w') as f:
+                json.dump(self.val_results, f)
diff --git a/examples/hello-tf2/config/config_fed_client.json b/examples/hello-tf2/config/config_fed_client.json
index f59b98ac37..ed260204c4 100644
--- a/examples/hello-tf2/config/config_fed_client.json
+++ b/examples/hello-tf2/config/config_fed_client.json
@@ -1,22 +1,34 @@
 {
-  "format_version": 1,
-  "platform": "TF2",
-  "client": {
-    "outbound_filters": [
-      {
-        "path": "filter.ExcludeVars",
+  "format_version": 2,
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "trainer.SimpleTrainer",
         "args": {
-          "exclude_vars": ["flatten"]
+          "epochs_per_round": 2
         }
       }
-    ],
-    "inbound_filters": []
-  },
-  "client_trainer": {
-    "path": "trainer.SimpleTrainer",
-    "args": {
-      "epochs_per_round": 2
     }
-  },
-  "handlers": []
-}
+  ],
+  "task_result_filters": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "filters": [
+        {
+          "path": "filter.ExcludeVars",
+          "args": {
+            "exclude_vars": [
+              "flatten"
+            ]
+          }
+        }
+      ]
+    }
+  ],
+  "task_data_filters": []
+}
\ No newline at end of file
diff --git a/examples/hello-tf2/config/config_fed_server.json b/examples/hello-tf2/config/config_fed_server.json
index fe69f250bc..5a8dcba9b4 100644
--- a/examples/hello-tf2/config/config_fed_server.json
+++ b/examples/hello-tf2/config/config_fed_server.json
@@ -1,29 +1,46 @@
 {
-  "format_version": 1,
-  "servers": [
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "tf2_model_persistor.TF2ModelPersistor",
+      "args": {
+        "save_name": "tf2weights.pickle"
+      }
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
     {
-      "min_num_clients": 1,
-      "max_num_clients": 100,
-      "wait_after_min_clients": 10,
-      "heart_beat_timeout": 600,
-      "start_round": 0,
-      "num_rounds": 3
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.accumulate_model_aggregator.AccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
     }
   ],
-  "aggregator": {
-    "path": "model_aggregator.AccumulateAggregator",
-    "args": {}
-  },
-  "outbound_filters": [],
-  "inbound_filters": [],
-  "persistor": {
-    "path": "model_persistor.TF2ModelPersistor",
-    "args": {
-      "save_name": "tf2weights.pickle"
+  "workflows": [
+    {
+      "id": "scatter_gather_ctl",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 1,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 0
+      }
     }
-  },
-  "shareable_generator": {
-    "name": "FullModelShareableGenerator"
-  }
-}
-
+  ]
+}
\ No newline at end of file
diff --git a/examples/hello-tf2/custom/__init__.py b/examples/hello-tf2/custom/__init__.py
index e69de29bb2..3291348d45 100644
--- a/examples/hello-tf2/custom/__init__.py
+++ b/examples/hello-tf2/custom/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/examples/hello-tf2/custom/filter.py b/examples/hello-tf2/custom/filter.py
index b655d189ea..056a0265bc 100644
--- a/examples/hello-tf2/custom/filter.py
+++ b/examples/hello-tf2/custom/filter.py
@@ -1,7 +1,22 @@
-import numpy as np
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
 
+import numpy as np
+from nvflare.apis.dxo import DXO, DataKind, from_shareable
 from nvflare.apis.filter import Filter
-from nvflare.apis.fl_constant import FLConstants, ShareableKey, ShareableValue
 from nvflare.apis.fl_context import FLContext
 from nvflare.apis.shareable import Shareable
 
@@ -13,38 +28,98 @@ class ExcludeVars(Filter):
     Args:
         exclude_vars: if not specified (None), all layers are being encrypted;
                       if list of variable/layer names, only specified variables are excluded;
+                      if string containing regular expression (e.g. "conv"), only matched variables are being excluded.
     """
 
     def __init__(self, exclude_vars=None):
         super().__init__()
         self.exclude_vars = exclude_vars
-
+        self.skip = False
         if self.exclude_vars is not None:
-            assert isinstance(self.exclude_vars, list), (
-                "Provide a list of layer names "
-            )
-            for var in self.exclude_vars:
-                assert isinstance(var, str), (
-                    "encrypt_layers needs to be a list of layer names to encrypt."
+            if not (
+                isinstance(self.exclude_vars, list)
+                or isinstance(self.exclude_vars, str)
+            ):
+                self.skip = True
+                self.logger.debug(
+                    "Need to provide a list of layer names or a string for regex matching"
+                )
+                return
+
+            if isinstance(self.exclude_vars, list):
+                for var in self.exclude_vars:
+                    if not isinstance(var, str):
+                        self.skip = True
+                        self.logger.debug(
+                            "encrypt_layers needs to be a list of layer names to encrypt."
+                        )
+                        return
+                self.logger.debug(f"Excluding {self.exclude_vars} from shareable")
+            elif isinstance(self.exclude_vars, str):
+                self.exclude_vars = (
+                    re.compile(self.exclude_vars) if self.exclude_vars else None
+                )
+                if self.exclude_vars is None:
+                    self.skip = True
+                self.logger.debug(
+                    f'Excluding all layers based on regex matches with "{self.exclude_vars}"'
                 )
-            self.logger.info(f"Excluding {self.exclude_vars} from shareable")
         else:
-            self.logger.info(f"Not excluding anything")
+            self.logger.debug("Not excluding anything")
+            self.skip = True
 
     def process(self, shareable: Shareable, fl_ctx: FLContext) -> Shareable:
-        weights = shareable[ShareableKey.MODEL_WEIGHTS]
 
-        # set excluded variables to all zeros
+        self.log_debug(fl_ctx, "inside filter")
+        if self.skip:
+            return shareable
+
+        try:
+            dxo = from_shareable(shareable)
+        except:
+            self.log_exception(fl_ctx, "shareable data is not a valid DXO")
+            return shareable
+
+        assert isinstance(dxo, DXO)
+        if dxo.data_kind not in (DataKind.WEIGHT_DIFF, DataKind.WEIGHTS):
+            self.log_debug(fl_ctx, "I cannot handle {}".format(dxo.data_kind))
+            return shareable
+
+        if dxo.data is None:
+            self.log_debug(fl_ctx, "no data to filter")
+            return shareable
+
+        weights = dxo.data
+
+        # parse regex encrypt layers
+        if isinstance(self.exclude_vars, re.Pattern):
+            re_pattern = self.exclude_vars
+            self.exclude_vars = []
+            for var_name in weights.keys():
+                if re_pattern.search(var_name):
+                    self.exclude_vars.append(var_name)
+            self.log_debug(fl_ctx, f"Regex found {self.exclude_vars} matching layers.")
+            if len(self.exclude_vars) == 0:
+                self.log_warning(
+                    fl_ctx, f"No matching layers found with regex {re_pattern}"
+                )
+
+        # remove variables
         n_excluded = 0
-        var_names = list(weights.keys()) 
+        var_names = list(
+            weights.keys()
+        )  # needs to recast to list to be used in for loop
         n_vars = len(var_names)
         for var_name in var_names:
-            if self.exclude_vars:
-                if var_name in self.exclude_vars:
-                    self.logger.info(f"Excluding {var_name}")
-                    weights[var_name] = np.zeros(weights[var_name].shape)        
-                    n_excluded += 1
-        self.logger.info(f"Excluded {n_excluded} of {n_vars} variables")
-
-        shareable[ShareableKey.MODEL_WEIGHTS] = weights
-        return shareable
+            # self.logger.info(f"Checking {var_name}")
+            if var_name in self.exclude_vars:
+                self.log_debug(fl_ctx, f"Excluding {var_name}")
+                weights[var_name] = np.zeros(weights[var_name].shape)
+                n_excluded += 1
+        self.log_debug(
+            fl_ctx,
+            f"Excluded {n_excluded} of {n_vars} variables. {len(weights.keys())} remaining.",
+        )
+
+        dxo.data = weights
+        return dxo.update_shareable(shareable)
diff --git a/examples/hello-tf2/custom/model_aggregator.py b/examples/hello-tf2/custom/model_aggregator.py
deleted file mode 100644
index d24dd5d91b..0000000000
--- a/examples/hello-tf2/custom/model_aggregator.py
+++ /dev/null
@@ -1,80 +0,0 @@
-from typing import Tuple
-import numpy as np
-
-from nvflare.apis.aggregator import Aggregator
-from nvflare.apis.fl_constant import FLConstants, ShareableKey, ShareableValue
-from nvflare.apis.fl_context import FLContext
-from nvflare.apis.shareable import Shareable
-from nvflare.utils.fl_ctx_sanity_check import server_fl_ctx_sanity_check
-
-
-class AccumulateAggregator(Aggregator):
-    def __init__(self, exclude_vars=None, aggregation_weights=None):
-        super().__init__()
-        self.accumulator = []
-
-    def accept(self, shareable: Shareable, fl_ctx: FLContext) -> Tuple[bool, bool]:
-        """Store shareable and update aggregator's internal state
-        Args:
-            shareable: information from client
-            fl_ctx: context provided by workflow
-
-        Returns:
-            The first boolean indicates if this shareable is accepted.
-            The second boolean indicates if aggregate can be called.
-        """
-
-        server_fl_ctx_sanity_check(fl_ctx)
-        current_round = fl_ctx.get_prop(FLConstants.CURRENT_ROUND)
-        shared_fl_context = fl_ctx.get_prop(FLConstants.PEER_CONTEXT)
-        client_name = shared_fl_context.get_prop(FLConstants.CLIENT_NAME)
-        contribution_round = shared_fl_context.get_prop(FLConstants.CURRENT_ROUND)
-
-        accepted = False
-        if contribution_round == current_round and not self._client_in_accumulator(client_name):
-            self.accumulator.append(shared_fl_context)
-            accepted = True
-
-        return accepted, False
-
-    def _client_in_accumulator(self, client_name):
-        for item in self.accumulator:
-            if client_name == item.get_prop(FLConstants.CLIENT_NAME):
-                return True
-        return False
-
-    def aggregate(self, fl_ctx: FLContext) -> Shareable:
-        """Called when workflow determines to generate shareable to send back to clients
-
-        Args:
-            fl_ctx (FLContext): context provided by workflow
-
-        Returns:
-            Shareable: the average of accepted shareables from clients
-        """
-        
-        server_fl_ctx_sanity_check(fl_ctx)
-        current_round = fl_ctx.get_prop(FLConstants.CURRENT_ROUND)
-        self.logger.info("aggregating %s updates at round %s", len(self.accumulator), current_round)
-
-        vars_to_aggregate = self.accumulator[0].get_prop(FLConstants.SHAREABLE)[ShareableKey.MODEL_WEIGHTS].keys()
-
-        # perform average of clients' weights
-        aggregated_model = {}
-        for v_name in vars_to_aggregate:
-            np_vars = []
-            for client_ctx in self.accumulator:
-                data = client_ctx.get_prop(FLConstants.SHAREABLE)[ShareableKey.MODEL_WEIGHTS]
-                np_vars.append(data[v_name])
-
-            new_val = np.sum(np_vars, axis=0) / len(self.accumulator)
-            aggregated_model[v_name] = new_val
-
-        self.accumulator.clear()
-
-        shareable = Shareable()
-        shareable[ShareableKey.TYPE] = ShareableValue.TYPE_WEIGHTS
-        shareable[ShareableKey.DATA_TYPE] = ShareableValue.DATA_TYPE_UNENCRYPTED
-        shareable[ShareableKey.MODEL_WEIGHTS] = aggregated_model
-        return shareable
-
diff --git a/examples/hello-tf2/custom/model_persistor.py b/examples/hello-tf2/custom/model_persistor.py
deleted file mode 100644
index 0e89403d53..0000000000
--- a/examples/hello-tf2/custom/model_persistor.py
+++ /dev/null
@@ -1,69 +0,0 @@
-import os
-import pickle
-import numpy as np
-
-import tensorflow as tf
-from net import Net
-from nvflare.apis.fl_context import FLContext
-from nvflare.apis import Model, ModelPersistor
-from nvflare.apis.fl_constant import FLConstants
-
-
-class TF2ModelPersistor(ModelPersistor):
-    def __init__(self, save_name=None):
-        super().__init__()
-        self.save_name = save_name
-
-    def _initialize(self, fl_ctx: FLContext):
-        # get save path from FLContext
-        train_root = fl_ctx.get_prop(FLConstants.TRAIN_ROOT)
-        log_dir = fl_ctx.get_prop(FLConstants.LOG_DIR)
-        if log_dir:
-            self.log_dir = os.path.join(train_root, log_dir)
-        else:
-            self.log_dir = train_root
-        self._pkl_save_path = os.path.join(self.log_dir, self.save_name)
-        if not os.path.exists(self.log_dir):
-            os.makedirs(self.log_dir)
-
-    def load_model(self, fl_ctx: FLContext) -> Model:
-        """
-            initialize and load the Model.
-
-        Args:
-            fl_ctx: FLContext
-
-        Returns:
-            Model object
-        """
-        self._initialize(fl_ctx)
-
-        model = Model()
-        if os.path.exists(self._pkl_save_path):
-            self.logger.info(f"Loading server weights")
-            with open(self._pkl_save_path, "rb") as f:
-                var_dict = pickle.load(f)
-        else:
-            self.logger.info(f"Initializing server model")
-            network = Net()
-            loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
-            network.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
-            _ = network(tf.keras.Input(shape=(28, 28)))
-            var_dict = {
-                network.get_layer(index=key).name: value
-                for key, value in enumerate(network.get_weights())
-            }
-        model.update(var_dict)
-        return model
-
-    def save_model(self, model: Model, fl_ctx: FLContext):
-        """
-            persist the Model object
-
-        Args:
-            model: Model object
-            fl_ctx: FLContext
-        """
-        self.logger.info(f"Saving aggregated server weights: \n {model}")
-        with open(self._pkl_save_path, "wb") as f:
-            pickle.dump(model, f)
diff --git a/examples/hello-tf2/custom/net.py b/examples/hello-tf2/custom/net.py
deleted file mode 100644
index 817ffcf0b2..0000000000
--- a/examples/hello-tf2/custom/net.py
+++ /dev/null
@@ -1,17 +0,0 @@
-import tensorflow as tf
-
-
-class Net(tf.keras.Model):
-    def __init__(self):
-        super().__init__()
-        self.flatten = tf.keras.layers.Flatten(input_shape=(28, 28))
-        self.dense1 = tf.keras.layers.Dense(128, activation="relu")
-        self.dropout = tf.keras.layers.Dropout(0.2)
-        self.dense2 = tf.keras.layers.Dense(10)
-
-    def call(self, x):
-        x = self.flatten(x)
-        x = self.dense1(x)
-        x = self.dropout(x)
-        x = self.dense2(x)
-        return x
diff --git a/examples/hello-tf2/custom/tf2_model_persistor.py b/examples/hello-tf2/custom/tf2_model_persistor.py
new file mode 100644
index 0000000000..8fb3453bbd
--- /dev/null
+++ b/examples/hello-tf2/custom/tf2_model_persistor.py
@@ -0,0 +1,114 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pickle
+import json
+
+import tensorflow as tf
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from tf2_net import Net
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.abstract.model import make_model_learnable
+
+
+class TF2ModelPersistor(ModelPersistor):
+    def __init__(self, save_name="tf2_model.pkl"):
+        super().__init__()
+        self.save_name = save_name
+
+    def _initialize(self, fl_ctx: FLContext):
+        # get save path from FLContext
+        app_root = fl_ctx.get_prop(FLContextKey.APP_ROOT)
+        env = None
+        run_args = fl_ctx.get_prop(FLContextKey.ARGS)
+        if run_args:
+            env_config_file_name = os.path.join(app_root, run_args.env)
+            if os.path.exists(env_config_file_name):
+                try:
+                    with open(env_config_file_name) as file:
+                        env = json.load(file)
+                except:
+                    self.system_panic(
+                        reason="error opening env config file {}".format(env_config_file_name), fl_ctx=fl_ctx
+                    )
+                    return
+
+        if env is not None:
+            if env.get("APP_CKPT_DIR", None):
+                fl_ctx.set_prop(AppConstants.LOG_DIR, env["APP_CKPT_DIR"], private=True, sticky=True)
+            if env.get("APP_CKPT") is not None:
+                fl_ctx.set_prop(
+                    AppConstants.CKPT_PRELOAD_PATH,
+                    env["APP_CKPT"],
+                    private=True,
+                    sticky=True,
+                )
+
+        log_dir = fl_ctx.get_prop(AppConstants.LOG_DIR)
+        if log_dir:
+            self.log_dir = os.path.join(app_root, log_dir)
+        else:
+            self.log_dir = app_root
+        self._pkl_save_path = os.path.join(self.log_dir, self.save_name)
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
+
+        fl_ctx.sync_sticky()
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        """
+            initialize and load the Model.
+
+        Args:
+            fl_ctx: FLContext
+
+        Returns:
+            Model object
+        """
+
+        if os.path.exists(self._pkl_save_path):
+            self.logger.info(f"Loading server weights")
+            with open(self._pkl_save_path, "rb") as f:
+                model_learnable = pickle.load(f)
+        else:
+            self.logger.info(f"Initializing server model")
+            network = Net()
+            loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+            network.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
+            _ = network(tf.keras.Input(shape=(28, 28)))
+            var_dict = {network.get_layer(index=key).name: value for key, value in enumerate(network.get_weights())}
+            model_learnable = make_model_learnable(var_dict, dict())
+        return model_learnable
+
+    def handle_event(self, event: str, fl_ctx: FLContext):
+        if event == EventType.START_RUN:
+            self._initialize(fl_ctx)
+
+    def save_model(self, model_learnable: ModelLearnable, fl_ctx: FLContext):
+        """
+            persist the Model object
+
+        Args:
+            model: Model object
+            fl_ctx: FLContext
+        """
+        model_learnable_info = {k: str(type(v)) for k, v in model_learnable.items()}
+        self.logger.info(f"Saving aggregated server weights: \n {model_learnable_info}")
+        with open(self._pkl_save_path, "wb") as f:
+            pickle.dump(model_learnable, f)
diff --git a/examples/hello-tf2/custom/tf2_net.py b/examples/hello-tf2/custom/tf2_net.py
new file mode 100644
index 0000000000..526eeea0ca
--- /dev/null
+++ b/examples/hello-tf2/custom/tf2_net.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tensorflow as tf
+
+
+class Net(tf.keras.Model):
+    def __init__(self):
+        super().__init__()
+        self.flatten = tf.keras.layers.Flatten(input_shape=(28, 28))
+        self.dense1 = tf.keras.layers.Dense(128, activation="relu")
+        self.dropout = tf.keras.layers.Dropout(0.2)
+        self.dense2 = tf.keras.layers.Dense(10)
+
+    def call(self, x):
+        x = self.flatten(x)
+        x = self.dense1(x)
+        x = self.dropout(x)
+        x = self.dense2(x)
+        return x
diff --git a/examples/hello-tf2/custom/trainer.py b/examples/hello-tf2/custom/trainer.py
index cfdb89af9e..2355836207 100644
--- a/examples/hello-tf2/custom/trainer.py
+++ b/examples/hello-tf2/custom/trainer.py
@@ -1,15 +1,32 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import tensorflow as tf
 import numpy as np
 
-from net import Net
+from nvflare.apis.dxo import DXO, DataKind, from_shareable
+from nvflare.apis.fl_constant import ReturnCode
 from nvflare.apis.event_type import EventType
-from nvflare.apis.fl_constant import FLConstants, ShareableKey, ShareableValue
+from nvflare.apis.executor import Executor
 from nvflare.apis.fl_context import FLContext
-from nvflare.apis.shareable import Shareable
-from nvflare.apis.trainer import Trainer
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.apis.signal import Signal
+
+from tf2_net import Net
 
 
-class SimpleTrainer(Trainer):
+class SimpleTrainer(Executor):
     def __init__(self, epochs_per_round):
         super().__init__()
         self.epochs_per_round = epochs_per_round
@@ -21,7 +38,7 @@ def handle_event(self, event_type: str, fl_ctx: FLContext):
         if event_type == EventType.START_RUN:
             self.setup(fl_ctx)
 
-    def setup(self, fl_ctx):
+    def setup(self, fl_ctx: FLContext):
         (self.train_images, self.train_labels), (
             self.test_images,
             self.test_labels,
@@ -32,27 +49,36 @@ def setup(self, fl_ctx):
         )
 
         # simulate separate datasets for each client by dividing MNIST dataset in half
-        client_name = fl_ctx.get_prop(FLConstants.CLIENT_NAME)
+        client_name = fl_ctx.get_identity_name()
         if client_name == "site-1":
-            self.train_images = self.train_images[:len(self.train_images)//2]
-            self.train_labels = self.train_labels[:len(self.train_labels)//2]
-            self.test_images = self.test_images[:len(self.test_images)//2]
-            self.test_labels = self.test_labels[:len(self.test_labels)//2] 
+            self.train_images = self.train_images[: len(self.train_images) // 2]
+            self.train_labels = self.train_labels[: len(self.train_labels) // 2]
+            self.test_images = self.test_images[: len(self.test_images) // 2]
+            self.test_labels = self.test_labels[: len(self.test_labels) // 2]
         elif client_name == "site-2":
-            self.train_images = self.train_images[len(self.train_images)//2:]
-            self.train_labels = self.train_labels[len(self.train_labels)//2:]
-            self.test_images = self.test_images[len(self.test_images)//2:]
-            self.test_labels = self.test_labels[len(self.test_labels)//2:] 
-        
+            self.train_images = self.train_images[len(self.train_images) // 2 :]
+            self.train_labels = self.train_labels[len(self.train_labels) // 2 :]
+            self.test_images = self.test_images[len(self.test_images) // 2 :]
+            self.test_labels = self.test_labels[len(self.test_labels) // 2 :]
+
         model = Net()
 
         loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
         model.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
         _ = model(tf.keras.Input(shape=(28, 28)))
-
+        self.var_list = [
+            model.get_layer(index=index).name
+            for index in range(len(model.get_weights()))
+        ]
         self.model = model
 
-    def train(self, shareable: Shareable, fl_ctx: FLContext, abort_signal) -> Shareable:
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
         """
         This function is an extended function from the super class.
         As a supervised learning based trainer, the train function will run
@@ -61,6 +87,7 @@ def train(self, shareable: Shareable, fl_ctx: FLContext, abort_signal) -> Sharea
         to server for aggregation.
 
         Args:
+            task_name: dispatched task
             shareable: the `Shareable` object acheived from server.
             fl_ctx: the `FLContext` object achieved from server.
             abort_signal: if triggered, the training will be aborted.
@@ -70,18 +97,29 @@ def train(self, shareable: Shareable, fl_ctx: FLContext, abort_signal) -> Sharea
         """
 
         # retrieve model weights download from server's shareable
-        model_weights = shareable[ShareableKey.MODEL_WEIGHTS]
+        if abort_signal.triggered:
+            return make_reply(ReturnCode.OK)
+
+        if task_name != "train":
+            return shareable
+
+        dxo = from_shareable(shareable)
+        model_weights = dxo.data
 
         # use previous round's client weights to replace excluded layers from server
         prev_weights = {
-            self.model.get_layer(index=key).name: value for key, value in enumerate(self.model.get_weights())
+            self.model.get_layer(index=key).name: value
+            for key, value in enumerate(self.model.get_weights())
         }
-        for key, value in model_weights.items():
+        print("dxo")
+        ordered_model_weights = {key: model_weights.get(key) for key in prev_weights}
+        for key in self.var_list:
+            value = ordered_model_weights.get(key)
             if np.all(value == 0):
-                model_weights[key] = prev_weights[key]
+                ordered_model_weights[key] = prev_weights[key]
 
         # update local model weights with received weights
-        self.model.set_weights(list(model_weights.values()))
+        self.model.set_weights(list(ordered_model_weights.values()))
 
         # adjust LR or other training time info as needed
         # such as callback in the fit function
@@ -93,11 +131,12 @@ def train(self, shareable: Shareable, fl_ctx: FLContext, abort_signal) -> Sharea
         )
 
         # report updated weights in shareable
-        shareable = Shareable()
-        shareable[ShareableKey.TYPE] = ShareableValue.TYPE_WEIGHTS
-        shareable[ShareableKey.DATA_TYPE] = ShareableValue.DATA_TYPE_UNENCRYPTED
-        shareable[ShareableKey.MODEL_WEIGHTS] = {
-            self.model.get_layer(index=key).name: value for key, value in enumerate(self.model.get_weights())
+        weights = {
+            self.model.get_layer(index=key).name: value
+            for key, value in enumerate(self.model.get_weights())
         }
-        self.logger.info(f"Sending shareable to server: \n {shareable[ShareableKey.MODEL_WEIGHTS]}")
-        return shareable
+        dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights)
+
+        self.log_info(fl_ctx, "Local epochs finished. Returning shareable")
+
+        return dxo.update_shareable(shareable)
diff --git a/nvflare/__init__.py b/nvflare/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/apis/__init__.py b/nvflare/apis/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/apis/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/apis/analytix.py b/nvflare/apis/analytix.py
new file mode 100644
index 0000000000..4f19231ed7
--- /dev/null
+++ b/nvflare/apis/analytix.py
@@ -0,0 +1,48 @@
+from enum import Enum
+
+from nvflare.apis.dxo import DXO, DataKind
+
+DATA_TYPE_KEY = "analytics_data_type"
+KWARGS_KEY = "analytics_kwargs"
+
+
+class DataType(Enum):
+    SCALARS = "SCALARS"
+    SCALAR = "SCALAR"
+    IMAGE = "IMAGE"
+    TEXT = "TEXT"
+
+
+class Data:
+    """This class defines Data format.
+    It is a wrapper to provide from / to DXO conversion.
+    """
+
+    def __init__(self, tag, value, data_type, kwargs):
+        self.tag = tag
+        self.value = value
+        self.data_type = data_type
+        self.kwargs = kwargs
+
+    def to_dxo(self):
+        dxo = DXO(data_kind=DataKind.ANALYTIC, data={self.tag: self.value})
+        dxo.set_meta_prop(DATA_TYPE_KEY, self.data_type)
+        dxo.set_meta_prop(KWARGS_KEY, self.kwargs)
+        return dxo
+
+    @classmethod
+    def from_dxo(cls, dxo: DXO):
+        if not isinstance(dxo, DXO):
+            raise TypeError(f"dxo is not of type DXO, instead it has type {type(dxo)}.")
+
+        if len(dxo.data) != 1:
+            raise ValueError("dxo does not have the correct format for AnalyticsData.")
+
+        tag, value = list(dxo.data.items())[0]
+
+        data_type = dxo.get_meta_prop(DATA_TYPE_KEY)
+        kwargs = dxo.get_meta_prop(KWARGS_KEY)
+        if not isinstance(data_type, DataType):
+            raise ValueError("data_type is not supported.")
+
+        return cls(tag, value, data_type, kwargs)
diff --git a/nvflare/apis/app_validation.py b/nvflare/apis/app_validation.py
new file mode 100644
index 0000000000..bfcfb168e2
--- /dev/null
+++ b/nvflare/apis/app_validation.py
@@ -0,0 +1,42 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+
+class AppValidationKey(object):
+
+    BYOC = "byoc"
+    CUSTOM_DATA_LIST = "custom_datalist"
+
+
+class AppValidator(ABC):
+    @abstractmethod
+    def validate(self, app_folder: str) -> (str, {}):
+        """
+        Validate and/or clean the content of specified application folder.
+
+        Args:
+            app_folder: path to the app folder to be validated
+
+        Returns:
+            A tuple of (error_msg, authorization_context)
+
+            error_msg contains error message if failed to pass; otherwise an empty string.
+            authorization_context is the context needed by authorization.
+
+            For example: the result result could be ("", {"byoc": True, "custom_datalist": True})
+
+        """
+        pass
diff --git a/nvflare/apis/aux.py b/nvflare/apis/aux.py
new file mode 100644
index 0000000000..e5a6b6291d
--- /dev/null
+++ b/nvflare/apis/aux.py
@@ -0,0 +1,41 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+
+from .fl_context import FLContext
+from .shareable import Shareable
+
+
+class MessageSendStatus(enum.Enum):
+
+    OK = "ok"  # message sent and response received
+    TIMEOUT = "timeout"  # message sent but no response received
+    FAILURE = "failure"  # failed to send message
+    REPLY_ERROR = "reply_error"  # error in reply
+
+
+def aux_request_handle_func_signature(topic: str, request: Shareable, fl_ctx: FLContext) -> Shareable:
+    """
+    This is the signature of the message_handle_func.
+
+    Args:
+        topic: topic of the message to be handled
+        request: the message data to be handled
+        fl_ctx: FL context
+
+    Returns: a response to the requester
+
+    """
+    pass
diff --git a/nvflare/apis/client.py b/nvflare/apis/client.py
new file mode 100644
index 0000000000..cab6db7123
--- /dev/null
+++ b/nvflare/apis/client.py
@@ -0,0 +1,36 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+
+class Client:
+    def __init__(self, name, token) -> None:
+        # super().__init__()
+        self.name = name
+        self.token = token
+        self.last_connect_time = time.time()
+        self.props = {}
+
+    def set_token(self, token):
+        self.token = token
+
+    def get_token(self):
+        return self.token
+
+    def set_prop(self, name, value):
+        self.props[name] = value
+
+    def get_prop(self, name, default=None):
+        return self.props.get(name, default)
diff --git a/nvflare/apis/client_engine_spec.py b/nvflare/apis/client_engine_spec.py
new file mode 100644
index 0000000000..68b233119a
--- /dev/null
+++ b/nvflare/apis/client_engine_spec.py
@@ -0,0 +1,93 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+from nvflare.widgets.widget import Widget
+
+from .fl_context import FLContext
+from .shareable import Shareable
+from .workspace import Workspace
+
+
+class TaskAssignment(object):
+    def __init__(self, name: str, task_id: str, data: Shareable):
+        self.name = name
+        self.task_id = task_id
+        self.data = data
+        self.receive_time = time.time()
+
+
+class ClientEngineSpec(object):
+    def fire_event(self, event_type: str, fl_ctx: FLContext):
+        pass
+
+    def get_task_assignment(self, fl_ctx: FLContext) -> TaskAssignment:
+        pass
+
+    def new_context(self) -> FLContext:
+        # the engine must use FLContextManager to create a new context!
+        pass
+
+    def send_task_result(self, result: Shareable, fl_ctx: FLContext) -> bool:
+        pass
+
+    def get_workspace(self) -> Workspace:
+        pass
+
+    def get_component(self, component_id: str) -> object:
+        pass
+
+    def get_widget(self, widget_id: str) -> Widget:
+        pass
+
+    def register_aux_message_handler(self, topic: str, message_handle_func):
+        """
+        Register aux message handling function with specified topics.
+
+        Args:
+            topic: the topic to be handled by the func
+            message_handle_func: the func to handle the message. Must follow aux_message_handle_func_signature.
+
+        Returns:
+
+        Exception is raised when:
+            a handler is already registered for the topic;
+            bad topic - must be a non-empty string
+            bad message_handle_func - must be callable
+
+        Implementation Note:
+        This method should simply call the ClientAuxRunner's register_aux_message_handler method.
+        """
+        pass
+
+    def send_aux_request(self, topic: str, request: Shareable, timeout: float, fl_ctx: FLContext) -> Shareable:
+        """
+        Send a request to Server via the aux channel.
+
+        Args:
+            topic: topic of the request
+            request: request to be sent
+            timeout: number of secs to wait for replies. 0 means fire-and-forget.
+            fl_ctx: FL context
+
+        Returns: a reply Shareable
+
+        Implementation: simply calls the ClientAuxRunner's send_aux_request method.
+
+        """
+        pass
+
+    def fire_and_forget_aux_request(self, topic: str, request: Shareable, fl_ctx: FLContext) -> Shareable:
+        return self.send_aux_request(topic, request, 0.0, fl_ctx)
diff --git a/nvflare/apis/controller_spec.py b/nvflare/apis/controller_spec.py
new file mode 100644
index 0000000000..96f7451752
--- /dev/null
+++ b/nvflare/apis/controller_spec.py
@@ -0,0 +1,539 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+import uuid
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, List, Optional, Union
+
+from nvflare.apis.signal import Signal
+
+from .client import Client
+from .fl_context import FLContext
+from .shareable import Shareable
+
+
+class TaskCompletionStatus(Enum):
+
+    OK = "ok"
+    TIMEOUT = "timeout"
+    ERROR = "error"
+    CANCELLED = "cancelled"
+    ABORTED = "aborted"
+    IGNORED = "ignored"
+
+
+class Task(object):
+    def __init__(
+        self,
+        name: str,
+        data: Shareable,
+        props: Optional[Dict] = None,
+        timeout: int = 0,
+        before_task_sent_cb=None,
+        after_task_sent_cb=None,
+        result_received_cb=None,
+        task_done_cb=None,
+    ):
+        """
+
+        Args:
+            name (str): name of the task
+            data (Shareable): data of the task
+            props: Any additional properties of the task
+            timeout: How long this task will last. If == 0, the task never time out.
+            before_task_sent_cb: If provided, this callback would be called before controller sends the tasks to clients.
+                It needs to follow the before_task_sent_cb_signature.
+            after_task_sent_cb: If provided, this callback would be called after controller sends the tasks to clients.
+                It needs to follow the after_task_sent_cb_signature.
+            result_received_cb: If provided, this callback would be called when controller receives results from clients.
+                It needs to follow the result_received_cb_signature.
+            task_done_cb: If provided, this callback would be called when task is done.
+                It needs to follow the task_done_cb_signature.
+
+        """
+        if not isinstance(name, str):
+            raise TypeError("name must be an instance of str.")
+
+        if not isinstance(data, Shareable):
+            raise TypeError("data must be an instance of Shareable.")
+
+        self.name = name  # name of the task
+        self.data = data  # task data to be sent to client(s)
+        self.cb_lock = threading.Lock()
+
+        if props is None:
+            self.props = {}
+        else:
+            if not isinstance(props, dict):
+                raise TypeError("props must be None or dict.")
+            self.props = props
+
+        if not isinstance(timeout, int):
+            raise TypeError("timeout must be an instance of int.")
+
+        if timeout < 0:
+            raise ValueError("timeout must >= 0.")
+
+        if before_task_sent_cb is not None and not callable(before_task_sent_cb):
+            raise TypeError("before_task_sent must be a callable function.")
+
+        if after_task_sent_cb is not None and not callable(after_task_sent_cb):
+            raise TypeError("after_task_sent_cb must be a callable function.")
+
+        if result_received_cb is not None and not callable(result_received_cb):
+            raise TypeError("result_received must be a callable function.")
+
+        if task_done_cb is not None and not callable(task_done_cb):
+            raise TypeError("task_done must be a callable function.")
+
+        self.timeout = timeout
+        self.before_task_sent_cb = before_task_sent_cb
+        self.after_task_sent_cb = after_task_sent_cb
+        self.result_received_cb = result_received_cb
+        self.task_done_cb = task_done_cb
+
+        self.targets = None
+        self.client_tasks = []  # list of ClientTasks sent
+        self.last_client_task_map = {}  # dict of: client name => last ClientTask of the client
+        self.completion_status = None  # task completion status
+        self.is_standing = False  # whether the task is still standing
+        self.schedule_time = None  # when the task was scheduled
+        self.create_time = time.time()
+
+    def set_prop(self, key, value):
+        if key.startswith("__"):
+            raise ValueError("Keys start with __ is reserved. Please use other key.")
+        self.props[key] = value
+
+    def get_prop(self, key):
+        if key.startswith("__"):
+            raise ValueError("Keys start with __ is reserved. Please use other key.")
+        return self.props.get(key)
+
+
+class ClientTask(object):
+
+    """
+    ClientTask records the processing information of a task for a client.
+
+    """
+
+    def __init__(self, client: Client, task: Task):
+        self.client = client
+        self.task = task
+        self.id = str(uuid.uuid4())
+        self.task_send_count = 0  # number of times the task is sent to the client
+        self.task_sent_time = None  # last time the task was sent to the client
+        self.result_received_time = None  # time when the result was received from the client
+        self.result = None  # result submitted by the client, or processed result
+        self.props = {}  # callbacks can use this dict to keep additional processing info
+
+
+class SendOrder(Enum):
+
+    ANY = "any"
+    SEQUENTIAL = "sequential"
+
+
+def before_task_sent_cb_signature(client_task: ClientTask, fl_ctx: FLContext):
+    """
+    Signature of the before_task_sent CB.
+
+    Called before sending a task to a client.
+    Usually used to prepare the FL Context, which is created to process client's task req
+    You can also use this CB to alter the data of the task to be sent.
+
+    Args:
+        client_task: the client task that is about to be sent
+        fl_ctx: the FL context that comes with the client's task request.
+        Public properties you set to this context will be sent to the client!
+
+    Returns:
+
+    """
+    pass
+
+
+def after_task_sent_cb_signature(client_task: ClientTask, fl_ctx: FLContext):
+    """
+    Signature of the after_task_sent CB.
+
+    Called after sending a task to a client.
+    Usually used to clean up the FL Context or the Task data
+
+    Args:
+        client_task: the client task that has been sent
+        fl_ctx: the FL context that comes with the client's task request.
+
+    Returns:
+
+    """
+    pass
+
+
+def result_received_cb_signature(client_task: ClientTask, fl_ctx: FLContext):
+    """
+
+    Signature of result_received CB
+
+    Called after a result is received from a client
+
+    Args:
+        client_task: the client task that the result is for
+        fl_ctx: the FL context that comes with the client's result submission
+
+    Returns:
+
+    """
+    pass
+
+
+def task_done_cb_signature(task: Task, fl_ctx: FLContext):
+    """
+    Signature of task_done CB.
+
+    Called when the task is completed.
+
+    Args:
+        task: the task that is completed
+        fl_ctx: an instance of FL Context used for this call only.
+
+    Returns:
+
+    """
+    pass
+
+
+class ControllerSpec(ABC):
+    @abstractmethod
+    def start_controller(self, fl_ctx: FLContext):
+        """Starts the controller.
+
+        This method is called at the beginning of the RUN.
+
+        Args:
+            fl_ctx: the FL context. You can use this context to access services provided by the
+            framework. For example, you can get Command Register from it and register your
+            admin command modules.
+
+        Returns:
+
+        """
+        pass
+
+    @abstractmethod
+    def stop_controller(self, fl_ctx: FLContext):
+        """Stops the controller
+
+        This method is called right before the RUN is ended.
+
+        Args:
+            fl_ctx: the FL context. You can use this context to access services provided by the
+            framework. For example, you can get Command Register from it and unregister your
+            admin command modules.
+
+        Returns:
+
+        """
+        pass
+
+    @abstractmethod
+    def process_result_of_unknown_task(
+        self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext
+    ):
+        """
+        This is called when a result submission is received from a client, but no standing
+        task can be found for it (from the task queue)
+
+        This could happen when:
+        - the client's submission is too late - the task is already completed
+        - the Controller lost the task, e.g. the Server is restarted
+
+        Args:
+            client: the client that the result comes from
+            task_name: the name of the task
+            client_task_id: ID of the task
+            result: the result from the client
+            fl_ctx: the FL context that comes with the client's submission
+
+        Returns:
+
+        """
+        pass
+
+    def broadcast(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        min_responses: int = 0,
+        wait_time_after_min_received: int = 0,
+    ):
+        """
+        Schedule to broadcast the task to specified targets.
+        This is a non-blocking call.
+
+        The task is standing until one of the following conditions comes true:
+            - if timeout is specified (> 0), and the task has been standing for more than the specified time
+            - the controller has received the specified min_responses results for this task, and all target clients
+            are done.
+            - the controller has received the specified min_responses results for this task, and has waited
+            for wait_time_after_min_received.
+
+        While the task is standing:
+            - Before sending the task to a client, the before_task_sent CB (if specified) is called;
+            - When a result is received from a client, the result_received CB (if specified) is called;
+
+        After the task is done, the task_done CB (if specified) is called:
+            - If result_received CB is specified, the 'result' in the ClientTask of each
+            client is produced by the result_received CB;
+            - Otherwise, the 'result' contains the original result submitted by the clients;
+
+        NOTE: if the targets is None, the actual broadcast target clients will be dynamic, because the clients
+        could join/disconnect at any moment. While the task is standing, any client that joins automatically
+        becomes a target for this broadcast.
+
+        Args:
+            task: the task to be sent
+            fl_ctx: the FL context
+            targets: list of destination clients. None means all clients are determined dynamically;
+            min_responses: the min number of responses expected. If == 0, must get responses from
+            all clients that the task has been sent to;
+            wait_time_after_min_received: how long (secs) to wait after the min_responses is received.
+            If == 0, end the task immediately after the min responses are received;
+
+        Returns:
+
+        """
+        pass
+
+    def broadcast_and_wait(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        min_responses: int = 0,
+        wait_time_after_min_received: int = 0,
+        abort_signal: Signal = None,
+    ):
+        """
+        This is the blocking version of the 'broadcast' method.
+
+        First, the task is scheduled for broadcast (see the broadcast method);
+        It then waits until the task is completed.
+
+        Args:
+            task:
+            fl_ctx:
+            targets:
+            min_responses:
+            wait_time_after_min_received:
+            abort_signal: the abort signal. If triggered, this method stops waiting and returns to the caller.
+
+        Returns:
+
+        """
+        pass
+
+    def broadcast_forever(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+    ):
+        """
+        Schedule a broadcast task that never ends until timeout or explicitly cancelled.
+        All clients will get the task every time it asks for a new task.
+
+        This is a non-blocking call.
+
+        NOTE: you can change the content of the task in the before_task_sent function.
+
+        Args:
+            task:
+            fl_ctx:
+            targets:
+
+        Returns:
+
+        """
+        pass
+
+    def send(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        send_order: SendOrder = SendOrder.SEQUENTIAL,
+        task_assignment_timeout: int = 0,
+    ):
+        """
+        Schedule to send the task to a single target client.
+        This is a non-blocking call.
+
+        In ANY order, the target client is the first target that asks for task.
+        In SEQUENTIAL order, the controller will try its best to send the task to the first client
+        in the targets list. If can't, it will try the next target, and so on.
+
+        NOTE: if the 'targets' is None, the actual target clients will be dynamic, because the clients
+        could join/disconnect at any moment. While the task is standing, any client that joins automatically
+        becomes a target for this task.
+
+        If the send_order is SEQUENTIAL, the targets must be a non-empty list of client names.
+
+        Args:
+            task: the task to be sent
+            fl_ctx: the FL context
+            targets: list of candidate target clients.
+            send_order: how to choose the client to send the task.
+            task_assignment_timeout: in SEQUENTIAL order, this is the wait time for trying a target client, before trying next target.
+
+        Returns:
+
+        """
+        pass
+
+    def send_and_wait(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        send_order: SendOrder = SendOrder.SEQUENTIAL,
+        task_assignment_timeout: int = 0,
+        abort_signal: Signal = None,
+    ):
+        """This is the blocking version of the 'send' method.
+
+        First, the task is scheduled for send (see the 'send' method);
+        It then waits until the task is completed and returns the task completion status and collected result.
+
+        Args:
+            task:
+            fl_ctx:
+            targets:
+            send_order:
+            task_assignment_timeout:
+            abort_signal:
+
+        Returns:
+
+        """
+        pass
+
+    def relay(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        send_order: SendOrder = SendOrder.SEQUENTIAL,
+        task_assignment_timeout: int = 0,
+        task_result_timeout: int = 0,
+        dynamic_targets: bool = True,
+    ):
+        """Schedules a task to be done sequentially by the clients in the targets list. This is a non-blocking call.
+
+        Args:
+            task: the task to be performed by each client
+            fl_ctx: the FL context for scheduling the task
+            targets: list of clients. If None, all clients.
+            send_order: how to choose the next client
+            task_assignment_timeout: how long to wait for the expected client to get assigned
+            before assigning to next client.
+            task_result_timeout: how long to wait for result from the assigned client before giving up.
+            dynamic_targets: whether to dynamically grow the target list. If True, then the target list is
+            expanded dynamically when a new client joins.
+
+        Returns:
+
+        """
+        pass
+
+    def relay_and_wait(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        send_order=SendOrder.SEQUENTIAL,
+        task_assignment_timeout: int = 0,
+        task_result_timeout: int = 0,
+        dynamic_targets: bool = True,
+        abort_signal: Signal = None,
+    ):
+        """This is the blocking version of 'relay'."""
+        pass
+
+    def get_num_standing_tasks(self) -> int:
+        """Gets tasks that are currently standing.
+
+        Returns: length of the list of standing tasks
+
+        """
+        pass
+
+    def cancel_task(
+        self,
+        task: Task,
+        completion_status: TaskCompletionStatus = TaskCompletionStatus.CANCELLED,
+        fl_ctx: Optional[FLContext] = None,
+    ):
+        """Cancels the specified task.
+
+        If the task is standing, the task is cancelled immediately (and removed from job queue) and calls
+        the task_done CB (if specified);
+
+        If the task is not standing, this method has no effect.
+
+        Args:
+            task:
+            completion_status:
+            fl_ctx
+
+        Returns:
+
+        """
+        pass
+
+    def cancel_all_tasks(self, completion_status=TaskCompletionStatus.CANCELLED, fl_ctx: Optional[FLContext] = None):
+        """Cancels all standing tasks.
+
+        Returns:
+
+        """
+        pass
+
+    def abort_task(self, task: Task, fl_ctx: FLContext):
+        """Asks all clients to abort the execution of the specified task.
+
+        Args:
+            task: the task to be aborted
+            fl_ctx: the FL context
+
+        Returns:
+
+        """
+        pass
+
+    def abort_all_tasks(self, fl_ctx: FLContext):
+        """Asks clients to abort the execution of all tasks.
+
+        NOTE: the server should send a notification to all clients, regardless of whether the server
+        has any standing tasks.
+
+        Returns:
+
+        """
+        pass
diff --git a/nvflare/apis/dxo.py b/nvflare/apis/dxo.py
new file mode 100644
index 0000000000..aac0d27e46
--- /dev/null
+++ b/nvflare/apis/dxo.py
@@ -0,0 +1,138 @@
+import copy
+import pickle
+
+from nvflare.apis.shareable import ReservedHeaderKey, Shareable
+
+
+class DataKind(object):
+    WEIGHTS = "WEIGHTS"
+    WEIGHT_DIFF = "WEIGHT_DIFF"
+    METRICS = "METRICS"
+    MODEL = "MODEL"
+    ANALYTIC = "ANALYTIC"
+
+
+class MetaKey(object):
+
+    NUM_STEPS_CURRENT_ROUND = "NUM_STEPS_CURRENT_ROUND"
+    MODEL_OWNER = "MODEL_OWNER"
+    PROCESSED_ALGORITHM = "PROCESSED_ALGORITHM"
+    PROCESSED_KEYS = "PROCESSED_KEYS"
+    INITIAL_METRICS = "initial_metrics"
+
+
+_KEY_KIND = "kind"
+_KEY_DATA = "data"
+_KEY_META = "meta"
+_KEY_DXO = "DXO"
+
+
+class DXO(object):
+    def __init__(self, data_kind: str, data: dict, meta: dict = None):
+        """
+
+        Args:
+            data_kind:
+            data: clear-text data
+            meta: None or dict
+        """
+        self.data_kind = data_kind
+        self.data = data
+        self.meta = meta
+
+        err = self.validate()
+        if err:
+            raise ValueError("invalid DXO: {}".format(err))
+
+    def get_meta_prop(self, key: str, default=None):
+        if self.meta and isinstance(self.meta, dict):
+            return self.meta.get(key, default)
+        return default
+
+    def set_meta_prop(self, key: str, value):
+        if self.meta is None:
+            self.meta = {}
+        self.meta[key] = value
+
+    def remove_meta_props(self, keys: [str]):
+        if self.meta and keys:
+            for k in keys:
+                self.meta.pop(k, None)
+
+    def get_meta_props(self):
+        return self.meta
+
+    def update_meta_props(self, meta):
+        self.meta.update(copy.deepcopy(meta))
+
+    def _encode(self) -> dict:
+        return {_KEY_KIND: self.data_kind, _KEY_DATA: self.data, _KEY_META: self.meta}
+
+    def update_shareable(self, s: Shareable) -> Shareable:
+        s.set_header(key=ReservedHeaderKey.CONTENT_TYPE, value="DXO")
+
+        s[_KEY_DXO] = self._encode()
+        return s
+
+    def to_shareable(self) -> Shareable:
+        s = Shareable()
+        return self.update_shareable(s)
+
+    def to_bytes(self) -> bytes:
+        """method to serialize the DXO object into bytes.
+
+        Returns:
+            object serialized in bytes.
+
+        """
+        return pickle.dumps(self)
+
+    def validate(self) -> str:
+        if self.data is None:
+            return "missing data"
+
+        if not isinstance(self.data, dict):
+            return "invalid data: expect dict but got {}".format(type(self.data))
+
+        if self.meta is not None and not isinstance(self.meta, dict):
+            return "invalid props: expect dict but got {}".format(type(self.meta))
+
+        return ""
+
+
+def from_shareable(s: Shareable) -> DXO:
+    content_type = s.get_header(ReservedHeaderKey.CONTENT_TYPE)
+    if not content_type or content_type != "DXO":
+        raise ValueError("the shareable is not a valid DXO - expect content_type DXO but got {}".format(content_type))
+
+    encoded = s.get(_KEY_DXO, None)
+    if not encoded:
+        raise ValueError("the shareable is not a valid DXO - missing content")
+
+    if not isinstance(encoded, dict):
+        raise ValueError(
+            "the shareable is not a valid DXO - should be encoded as dict but got {}".format(type(encoded))
+        )
+
+    k = encoded.get(_KEY_KIND, None)
+    d = encoded.get(_KEY_DATA, None)
+    m = encoded.get(_KEY_META, None)
+
+    return DXO(data_kind=k, data=d, meta=m)
+
+
+def from_bytes(data: bytes) -> DXO:
+    """method to convert the object bytes into Model object.
+
+    Args:
+        data: a bytes object
+
+    Returns:
+        an object loaded by pickle from data
+
+    """
+    x = pickle.loads(data)
+    if isinstance(x, DXO):
+        return x
+    else:
+        raise ValueError(f"Data bytes are from type {type(x)} and do not represent a valid DXO instance.")
diff --git a/nvflare/apis/event_type.py b/nvflare/apis/event_type.py
new file mode 100644
index 0000000000..ed3b8e5ab2
--- /dev/null
+++ b/nvflare/apis/event_type.py
@@ -0,0 +1,47 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class EventType(object):
+    """
+    Built-in system events.
+    """
+
+    START_RUN = "_start_run"
+    END_RUN = "_end_run"
+    START_WORKFLOW = "_start_workflow"
+    END_WORKFLOW = "_end_workflow"
+    ABORT_TASK = "_abort_task"
+    FATAL_SYSTEM_ERROR = "_fatal_system_error"
+    FATAL_TASK_ERROR = "_fatal_task_error"
+
+    BEFORE_PULL_TASK = "_before_pull_task"
+    AFTER_PULL_TASK = "_after_pull_task"
+    BEFORE_PROCESS_SUBMISSION = "_before_process_submission"
+    AFTER_PROCESS_SUBMISSION = "_after_process_submission"
+
+    BEFORE_TASK_DATA_FILTER = "_before_task_data_filter"
+    AFTER_TASK_DATA_FILTER = "_after_task_data_filter"
+    BEFORE_TASK_RESULT_FILTER = "_before_task_result_filter"
+    AFTER_TASK_RESULT_FILTER = "_after_task_result_filter"
+    BEFORE_TASK_EXECUTION = "_before_task_execution"
+    AFTER_TASK_EXECUTION = "_after_task_execution"
+    BEFORE_SEND_TASK_RESULT = "_before_send_task_result"
+    AFTER_SEND_TASK_RESULT = "_after_send_task_result"
+
+    DEBUG_LOG_AVAILABLE = "_debug_log_available"
+    ERROR_LOG_AVAILABLE = "_error_log_available"
+    EXCEPTION_LOG_AVAILABLE = "_exception_log_available"
+    INFO_LOG_AVAILABLE = "_info_log_available"
+    WARNING_LOG_AVAILABLE = "_warning_log_available"
diff --git a/nvflare/apis/executor.py b/nvflare/apis/executor.py
new file mode 100644
index 0000000000..455e5ae5c0
--- /dev/null
+++ b/nvflare/apis/executor.py
@@ -0,0 +1,27 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+from nvflare.apis.signal import Signal
+
+from .fl_component import FLComponent
+from .fl_context import FLContext
+from .shareable import Shareable
+
+
+class Executor(FLComponent, ABC):
+    @abstractmethod
+    def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable:
+        pass
diff --git a/nvflare/apis/filter.py b/nvflare/apis/filter.py
new file mode 100644
index 0000000000..b0f0e00b52
--- /dev/null
+++ b/nvflare/apis/filter.py
@@ -0,0 +1,36 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+
+class Filter(FLComponent, ABC):
+    @abstractmethod
+    def process(self, shareable: Shareable, fl_ctx: FLContext) -> Shareable:
+        """
+            filter process apply to the Shareable object.
+
+        Args:
+            shareable: shareable
+            fl_ctx: FLContext
+
+        Returns:
+            a Shareable object
+
+        """
+        pass
diff --git a/nvflare/apis/fl_component.py b/nvflare/apis/fl_component.py
new file mode 100644
index 0000000000..af86f9f5f7
--- /dev/null
+++ b/nvflare/apis/fl_component.py
@@ -0,0 +1,163 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import traceback
+
+from nvflare.apis.utils.fl_context_utils import generate_log_message
+
+from .analytix import Data as AnalytixData
+from .analytix import DataType
+from .event_type import EventType
+from .fl_constant import EventScope, FedEventHeader, FLContextKey, LogMessageTag
+from .fl_context import FLContext
+from .shareable import Shareable
+
+
+class FLComponent(object):
+    def __init__(self):
+        self._name = self.__class__.__name__
+        self.logger = logging.getLogger(self._name)
+
+    def _fire(self, event_type: str, fl_ctx: FLContext):
+        fl_ctx.set_prop(FLContextKey.EVENT_ORIGIN, self._name, private=True, sticky=False)
+        engine = fl_ctx.get_engine()
+        if engine is None:
+            # must not call self.fire_event again, or it will be a dead loop!
+            self.log_error(fl_ctx=fl_ctx, msg="Logic Error: no engine in fl_ctx: {}".format(fl_ctx), fire_event=False)
+        else:
+            engine.fire_event(event_type, fl_ctx)
+
+    def fire_event(self, event_type: str, fl_ctx: FLContext):
+        if not isinstance(event_type, str):
+            raise TypeError("expect event_type to be str, but got {}".format(type(event_type)))
+
+        if not event_type:
+            raise ValueError("event_type must be specified")
+
+        if not isinstance(fl_ctx, FLContext):
+            raise TypeError("expect fl_ctx to be FLContext, but got {}".format(type(fl_ctx)))
+
+        fl_ctx.set_prop(FLContextKey.EVENT_SCOPE, value=EventScope.LOCAL, private=True, sticky=False)
+        self._fire(event_type, fl_ctx)
+
+    def fire_fed_event(self, event_type: str, event_data: Shareable, fl_ctx: FLContext, targets=None):
+        if not isinstance(fl_ctx, FLContext):
+            raise TypeError("expect fl_ctx to be FLContext, but got {}".format(type(fl_ctx)))
+
+        if not isinstance(event_data, Shareable):
+            raise TypeError("expect event_data to be Shareable, but got {}".format(type(event_data)))
+
+        event_data.set_header(key=FedEventHeader.TARGETS, value=targets)
+        fl_ctx.set_prop(FLContextKey.EVENT_DATA, event_data, private=True, sticky=False)
+        fl_ctx.set_prop(FLContextKey.EVENT_SCOPE, value=EventScope.FEDERATION, private=True, sticky=False)
+        self._fire(event_type, fl_ctx)
+
+    def system_panic(self, reason: str, fl_ctx: FLContext):
+        """
+        Signal a fatal condition that could cause the RUN to end
+        Args:
+            reason: reason for panic
+            fl_ctx: the FL context
+
+        Returns:
+
+        """
+        fl_ctx.set_prop(FLContextKey.EVENT_DATA, reason, private=True, sticky=False)
+        self.fire_event(EventType.FATAL_SYSTEM_ERROR, fl_ctx)
+
+    def task_panic(self, reason: str, fl_ctx: FLContext):
+        """
+        Signal a fatal condition that could cause the current task (on Client) to end
+        Args:
+            reason: reason for panic
+            fl_ctx: FL context
+        Returns:
+
+        """
+        fl_ctx.set_prop(FLContextKey.EVENT_DATA, reason, private=True, sticky=False)
+        self.fire_event(EventType.FATAL_TASK_ERROR, fl_ctx)
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        """
+            perform the handler process based on the event_type.
+
+        Args:
+            event_type: event type fired by workflow
+            fl_ctx: FLContext
+
+        """
+        pass
+
+    def make_log_message(self, fl_ctx: FLContext, msg: str):
+        return generate_log_message(fl_ctx, msg)
+
+    def log_info(self, fl_ctx: FLContext, msg: str, fire_event=True):
+        log_msg = self.make_log_message(fl_ctx, msg)
+        self.logger.info(log_msg)
+
+        if fire_event:
+            self._fire_log_event(
+                event_type=EventType.INFO_LOG_AVAILABLE, log_tag=LogMessageTag.INFO, log_msg=log_msg, fl_ctx=fl_ctx
+            )
+
+    def log_warning(self, fl_ctx: FLContext, msg: str, fire_event=True):
+        log_msg = self.make_log_message(fl_ctx, msg)
+        self.logger.warning(log_msg)
+        if fire_event:
+            self._fire_log_event(
+                event_type=EventType.WARNING_LOG_AVAILABLE,
+                log_tag=LogMessageTag.WARNING,
+                log_msg=log_msg,
+                fl_ctx=fl_ctx,
+            )
+
+    def log_error(self, fl_ctx: FLContext, msg: str, fire_event=True):
+        log_msg = self.make_log_message(fl_ctx, msg)
+        self.logger.error(log_msg)
+        if fire_event:
+            self._fire_log_event(
+                event_type=EventType.ERROR_LOG_AVAILABLE, log_tag=LogMessageTag.ERROR, log_msg=log_msg, fl_ctx=fl_ctx
+            )
+
+    def log_debug(self, fl_ctx: FLContext, msg: str, fire_event=True):
+        log_msg = self.make_log_message(fl_ctx, msg)
+        self.logger.debug(log_msg)
+        if fire_event:
+            self._fire_log_event(
+                event_type=EventType.DEBUG_LOG_AVAILABLE, log_tag=LogMessageTag.DEBUG, log_msg=log_msg, fl_ctx=fl_ctx
+            )
+
+    def log_exception(self, fl_ctx: FLContext, msg: str, fire_event=True):
+        log_msg = self.make_log_message(fl_ctx, msg)
+        self.logger.error(log_msg)
+        traceback.print_exc()
+
+        # post exception log event - don't do it for now since exception could go deep.
+        #
+        # if fire_event:
+        #     ex_text = traceback.format_exc()
+        #     ex_msg = "{}\n{}".format(log_msg, ex_text)
+        #     self._fire_log_event(
+        #         event_type=EventType.EXCEPTION_LOG_AVAILABLE,
+        #         log_tag=LogMessageTag.EXCEPTION,
+        #         log_msg=ex_msg,
+        #         fl_ctx=fl_ctx,
+        #     )
+
+    def _fire_log_event(self, event_type: str, log_tag: str, log_msg: str, fl_ctx: FLContext):
+        event_data = AnalytixData(tag=log_tag, value=log_msg, data_type=DataType.TEXT, kwargs=None)
+        dxo = event_data.to_dxo()
+        fl_ctx.set_prop(key=FLContextKey.EVENT_DATA, value=dxo.to_shareable(), private=True, sticky=False)
+        self.fire_event(event_type=event_type, fl_ctx=fl_ctx)
diff --git a/nvflare/apis/fl_constant.py b/nvflare/apis/fl_constant.py
new file mode 100644
index 0000000000..3eec0aa62d
--- /dev/null
+++ b/nvflare/apis/fl_constant.py
@@ -0,0 +1,169 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from enum import Enum
+
+
+class ReturnCode(object):
+
+    OK = "OK"
+    ERROR = "ERROR"
+    MISSING_PEER_CONTEXT = "MISSING_PEER_CONTEXT"
+    BAD_PEER_CONTEXT = "BAD_PEER_CONTEXT"
+    RUN_MISMATCH = "RUN_MISMATCH"
+    TASK_UNKNOWN = "TASK_UNKNOWN"
+    TASK_DATA_FILTER_ERROR = "TASK_DATA_FILTER_ERROR"
+    TASK_RESULT_FILTER_ERROR = "TASK_RESULT_FILTER_ERROR"
+    EXECUTION_EXCEPTION = "EXECUTION_EXCEPTION"
+    EXECUTION_RESULT_ERROR = "EXECUTION_RESULT_ERROR"
+    COMMUNICATION_ERROR = "COMMUNICATION_ERROR"
+    BAD_TASK_DATA = "BAD_TASK_DATA"
+    TOPIC_UNKNOWN = "TOPIC_UNKNOWN"
+    HANDLER_EXCEPTION = "HANDLER_EXCEPTION"
+    BAD_REQUEST_DATA = "BAD_TASK_DATA"
+    COMMUNICATION_ERROR = "COMMUNICATION_ERROR"
+
+
+class MachineStatus(Enum):
+    """
+    Status Lifecycle:
+
+        STOPPED <-> STARTING -> STARTED -> STOPPING -> STOPPED
+    """
+
+    STARTING = "starting"
+    STARTED = "started"
+    STOPPING = "stopping"
+    STOPPED = "stopped"
+
+
+class ReservedKey(object):
+
+    MANAGER = "__manager__"
+    ENGINE = "__engine__"
+    AUX_RUNNER = "__aux_runner__"
+    RUN_NUM = "__run_num__"
+    IDENTITY_NAME = "__identity_name__"  # identity of the endpoint (e.g. client name)
+    PEER_CTX = "__peer_ctx__"
+    RC = "__rc__"
+    COOKIE_JAR = "__cookie_jar__"
+    WORKSPACE_ROOT = "__workspace_root__"
+    APP_ROOT = "__app_root__"
+    CLIENT_NAME = "__client_name__"
+    TASK_NAME = "__task_name__"
+    TASK_DATA = "__task_data__"
+    TASK_RESULT = "__task_result__"
+    TASK_ID = "__task_id__"
+    EVENT_ID = "__event_id__"
+    IS_RESEND = "__is_resend__"
+    RUNNER = "__runner__"
+    WORKFLOW = "__workflow__"
+    REPLY = "__reply__"
+    EVENT_ORIGIN = "__event_origin__"
+    EVENT_ORIGIN_SITE = "__event_origin_site__"
+    EVENT_DATA = "__event_data__"
+    EVENT_SCOPE = "__event_scope__"
+    RUN_ABORT_SIGNAL = "__run_abort_signal__"
+    SHAREABLE = "__shareable__"
+    ARGS = "__args__"
+    WORKSPACE_OBJECT = "__workspace_object__"
+    RANK_NUMBER = "__rank_number__"
+    NUM_OF_PROCESSES = "__num_of_processes__"
+    FROM_RANK_NUMBER = "__from_rank_number__"
+    SECURE_MODE = "__secure_mode__"
+
+
+class FLContextKey(object):
+
+    TASK_NAME = ReservedKey.TASK_NAME
+    TASK_DATA = ReservedKey.TASK_DATA
+    TASK_RESULT = ReservedKey.TASK_RESULT
+    TASK_ID = ReservedKey.TASK_ID
+    EVENT_ID = ReservedKey.EVENT_ID
+    EVENT_ORIGIN = ReservedKey.EVENT_ORIGIN
+    EVENT_ORIGIN_SITE = ReservedKey.EVENT_ORIGIN_SITE
+    EVENT_DATA = ReservedKey.EVENT_DATA
+    EVENT_SCOPE = ReservedKey.EVENT_SCOPE
+    CLIENT_NAME = ReservedKey.CLIENT_NAME
+    WORKSPACE_ROOT = ReservedKey.WORKSPACE_ROOT
+    CURRENT_RUN = ReservedKey.RUN_NUM
+    APP_ROOT = ReservedKey.APP_ROOT
+    PEER_CONTEXT = ReservedKey.PEER_CTX
+    IS_CLIENT_TASK_RESEND = ReservedKey.IS_RESEND
+    RUNNER = ReservedKey.RUNNER
+    WORKFLOW = ReservedKey.WORKFLOW
+    SHAREABLE = ReservedKey.SHAREABLE
+    RUN_ABORT_SIGNAL = ReservedKey.RUN_ABORT_SIGNAL
+    ARGS = ReservedKey.ARGS
+    REPLY = ReservedKey.REPLY
+    WORKSPACE_OBJECT = ReservedKey.WORKSPACE_OBJECT
+    RANK_NUMBER = ReservedKey.RANK_NUMBER
+    NUM_OF_PROCESSES = ReservedKey.NUM_OF_PROCESSES
+    FROM_RANK_NUMBER = ReservedKey.FROM_RANK_NUMBER
+    SECURE_MODE = ReservedKey.SECURE_MODE
+
+
+class ReservedTopic(object):
+
+    END_RUN = "__end_run__"
+    ABORT_ASK = "__abort_task__"
+    AUX_COMMAND = "__aux_command__"
+
+
+class AdminCommandNames(object):
+
+    SET_RUN_NUMBER = "set_run_number"
+    DELETE_RUN_NUMBER = "delete_run_number"
+    DEPLOY_APP = "deploy_app"
+    START_APP = "start_app"
+    CHECK_STATUS = "check_status"
+    ABORT = "abort"
+    ABORT_TASK = "abort_task"
+    REMOVE_CLIENT = "remove_client"
+    SHUTDOWN = "shutdown"
+    RESTART = "restart"
+    SET_TIMEOUT = "set_timeout"
+    SHOW_STATS = "show_stats"
+    SHOW_ERRORS = "show_errors"
+    RESET_ERRORS = "reset_errors"
+    AUX_COMMAND = "aux_command"
+
+
+class FedEventHeader(object):
+
+    TIMESTAMP = "_timestamp"
+    EVENT_TYPE = "_event_type"
+    DIRECTION = "_direction"
+    ORIGIN = "_origin"
+    TARGETS = "_targets"
+
+
+class EventScope(object):
+
+    FEDERATION = "federation"
+    LOCAL = "local"
+
+
+class NonSerializableKeys(object):
+
+    KEYS = [ReservedKey.ENGINE, ReservedKey.MANAGER, ReservedKey.RUNNER]
+
+
+class LogMessageTag(object):
+
+    DEBUG = "log/debug"
+    ERROR = "log/error"
+    EXCEPTION = "log/exception"
+    INFO = "log/info"
+    WARNING = "log/warning"
diff --git a/nvflare/apis/fl_context.py b/nvflare/apis/fl_context.py
new file mode 100644
index 0000000000..6653e39ed3
--- /dev/null
+++ b/nvflare/apis/fl_context.py
@@ -0,0 +1,233 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import threading
+
+from .fl_constant import ReservedKey
+
+_update_lock = threading.Lock()
+
+
+class FLContext(object):
+    MASK_STICKY = 1 << 0
+    MASK_PRIVATE = 1 << 1
+
+    @classmethod
+    def _is_sticky(cls, mask) -> bool:
+        return mask & cls.MASK_STICKY > 0
+
+    @classmethod
+    def _is_private(cls, mask) -> bool:
+        return mask & cls.MASK_PRIVATE > 0
+
+    @classmethod
+    def _matching_private(cls, mask, private) -> bool:
+        return (mask & cls.MASK_PRIVATE > 0) == private
+
+    @classmethod
+    def _to_string(cls, mask) -> str:
+        if cls._is_private(mask):
+            result = "private:"
+        else:
+            result = "public:"
+
+        if cls._is_sticky(mask):
+            return result + "sticky"
+        else:
+            return result + "non-sticky"
+
+    def __init__(self):
+        self.model = None
+        self.props = {}
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def public_key_exists(self, key):
+        return key in self.props and not self._is_private(self.props[key]["mask"])
+
+    def get_all_public_props(self):
+        return {k: v["value"] for k, v in self.props.items() if not self._is_private(v["mask"])}
+
+    def set_prop(self, key: str, value, private=True, sticky=True):
+        if not isinstance(key, str):
+            raise ValueError("prop key must be str")
+
+        with _update_lock:
+            mask = 0
+            if private:
+                mask += FLContext.MASK_PRIVATE
+            if sticky:
+                mask += FLContext.MASK_STICKY
+            if key not in self.props or key in self.props and self.props[key]["mask"] == mask:
+                self.props[key] = {"value": value, "mask": mask}
+            else:
+                existing_mask = self.props[key]["mask"]
+                self.logger.warning(
+                    f"property {key} already exists with attributes "
+                    f"{self._to_string(existing_mask)}, cannot change to {self._to_string(mask)}"
+                )
+                return False
+        return True
+
+    def get_prop(self, key, default=None):
+        with _update_lock:
+            if key in self.props:
+                return self.props.get(key)["value"]
+            else:
+                return default
+
+    def get_prop_detail(self, key):
+        with _update_lock:
+            if key in self.props:
+                prop = self.props.get(key)
+                mask = prop["mask"]
+                return {"value": prop["value"], "private": self._is_private(mask), "sticky": self._is_sticky(mask)}
+            else:
+                return None
+
+    def remove_prop(self, key: str):
+        if not isinstance(key, str):
+            return
+
+        if key.startswith("__"):
+            # do not allow removal of reserved props!
+            return
+
+        with _update_lock:
+            self.props.pop(key, None)
+
+    def __str__(self):
+        raw_list = [f'{k}: {type(v["value"])}' for k, v in self.props.items()]
+        return " ".join(raw_list)
+
+    # some convenience methods
+    def get_engine(self):
+        return self.get_prop(key=ReservedKey.ENGINE, default=None)
+
+    def get_run_number(self):
+        return self.get_prop(key=ReservedKey.RUN_NUM, default=None)
+
+    def get_identity_name(self):
+        return self.get_prop(key=ReservedKey.IDENTITY_NAME, default="")
+
+    def get_run_abort_signal(self):
+        return self.get_prop(key=ReservedKey.RUN_ABORT_SIGNAL, default=None)
+
+    def set_peer_context(self, ctx):
+        self.set_prop(key=ReservedKey.PEER_CTX, value=ctx, private=True, sticky=False)
+
+    def get_peer_context(self):
+        return self.get_prop(key=ReservedKey.PEER_CTX, default=None)
+
+    def set_run_number(self, run_number):
+        return self.set_prop(key=ReservedKey.RUN_NUM, value=run_number, private=False)
+
+    def set_public_props(self, metadata: dict):
+        # remove all public props
+        self.props = {k: v for k, v in self.props.items() if self._is_private(v["mask"] or self._is_sticky(v["mask"]))}
+
+        for key, value in metadata.items():
+            self.set_prop(key, value, private=False, sticky=True)
+
+    def clone_sticky(self):
+        new_fl_ctx = FLContext()
+        for k, v in self.props.items():
+            if self._is_sticky(v["mask"]):
+                new_fl_ctx.props[k] = {"value": v["value"], "mask": v["mask"]}
+        return new_fl_ctx
+
+    def sync_sticky(self):
+        ctx_manager = self.get_prop(key=ReservedKey.MANAGER, default=None)
+        if not ctx_manager:
+            raise ValueError("FLContextManager does not exist.")
+
+        ctx_manager.finalize_context(self)
+
+    # implement Context Manager protocol
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        mgr = self.get_prop(key=ReservedKey.MANAGER)
+        # if not mgr:
+        #     raise ValueError("This FLContext is not managed. Please use FLContextManager to create FLContext")
+        if mgr:
+            mgr.finalize_context(self)
+
+
+class FLContextManager(object):
+
+    """
+    NOTE: The engine may create a new FLContextManager object for each RUN!
+
+    """
+
+    def __init__(self, engine, identity_name: str, run_num: int, public_stickers, private_stickers):
+        self.engine = engine
+        self.identity_name = identity_name
+        self.run_num = run_num
+        self._update_lock = threading.Lock()
+
+        self.public_stickers = {}
+        self.private_stickers = {}
+
+        if public_stickers and isinstance(public_stickers, dict):
+            self.public_stickers.update(public_stickers)
+
+        if private_stickers and isinstance(private_stickers, dict):
+            self.private_stickers.update(private_stickers)
+
+    def new_context(self) -> FLContext:
+        """Create a new FLContext object.
+
+        Sticky properties are copied from the stickers into the new context.
+
+        Returns: a FLContext object
+
+        """
+        ctx = FLContext()
+        ctx.set_prop(key=ReservedKey.MANAGER, value=self, private=True, sticky=False)
+
+        # set permanent props
+        ctx.set_prop(key=ReservedKey.ENGINE, value=self.engine, private=True)
+        ctx.set_prop(key=ReservedKey.RUN_NUM, value=self.run_num, private=False)
+
+        if self.identity_name:
+            ctx.set_prop(key=ReservedKey.IDENTITY_NAME, value=self.identity_name, private=False)
+
+        with self._update_lock:
+            for k, v in self.public_stickers.items():
+                ctx.set_prop(key=k, value=v, sticky=True, private=False)
+
+            for k, v in self.private_stickers.items():
+                ctx.set_prop(key=k, value=v, sticky=True, private=True)
+
+        return ctx
+
+    def finalize_context(self, ctx: FLContext):
+        """Finalize the context by copying/updating sticky props into stickers
+
+        Args:
+            ctx: the context to be finalized
+
+        Returns:
+
+        """
+        with self._update_lock:
+            for k, v in ctx.props.items():
+                if ctx._is_sticky(v["mask"]):
+                    if ctx._is_private(v["mask"]):
+                        self.private_stickers[k] = v["value"]
+                    else:
+                        self.public_stickers[k] = v["value"]
diff --git a/nvflare/apis/fl_exception.py b/nvflare/apis/fl_exception.py
new file mode 100644
index 0000000000..d133ead053
--- /dev/null
+++ b/nvflare/apis/fl_exception.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class FLCommunicationError(Exception):
+    """
+    Base class for fed_learn communication exceptions
+    """
+
+    def __init__(self, exception):
+        super().__init__()
+        # Copy all the gRPC exception properties into FLCommunicationError instance.
+        self.__dict__.update(exception.__dict__)
+
+
+class WorkflowError(Exception):
+    """
+    FL Workflow error to indicate not to continue workflow execution.
+    """
+
+    def __init__(self, *args: object) -> None:
+        super().__init__(*args)
diff --git a/nvflare/apis/impl/__init__.py b/nvflare/apis/impl/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/apis/impl/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/apis/impl/any_relay_manager.py b/nvflare/apis/impl/any_relay_manager.py
new file mode 100644
index 0000000000..d90b861a89
--- /dev/null
+++ b/nvflare/apis/impl/any_relay_manager.py
@@ -0,0 +1,155 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from typing import Tuple
+
+from nvflare.apis.controller_spec import ClientTask, Task, TaskCompletionStatus
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import ReservedHeaderKey, Shareable
+
+from .task_manager import TaskCheckStatus, TaskManager
+
+_KEY_DYNAMIC_TARGETS = "__dynamic_targets"
+_KEY_TASK_RESULT_TIMEOUT = "__task_result_timeout"
+_KEY_FINISHED_TARGETS = "__finished_targets"
+_KEY_PENDING_CLIENT = "__pending_client"
+
+
+class AnyRelayTaskManager(TaskManager):
+    def __init__(self, task: Task, task_result_timeout, dynamic_targets):
+        """Task manager for relay controller on SendOrder.ANY
+
+        Args:
+            task (Task): an instance of Task
+            task_result_timeout (int): timeout value on reply of one client
+            dynamic_targets (bool): allow clients to join after this task starts
+        """
+        TaskManager.__init__(self)
+
+        if task_result_timeout is None:
+            task_result_timeout = 0
+
+        task.props[_KEY_DYNAMIC_TARGETS] = dynamic_targets
+        task.props[_KEY_TASK_RESULT_TIMEOUT] = task_result_timeout
+        task.props[_KEY_FINISHED_TARGETS] = {}  # target name => times sent
+        task.props[_KEY_PENDING_CLIENT] = None
+
+    def check_task_send(self, client_task: ClientTask, fl_ctx: FLContext) -> TaskCheckStatus:
+        """Determine whether the task should be sent to the client.
+
+        Args:
+            client_task (ClientTask): the task processing state of the client
+            fl_ctx (FLContext): fl context that comes with the task request
+
+        Raises:
+            RuntimeError: when a client asking for a task while the same client_task has already been dispatched to it
+
+        Returns:
+            TaskCheckStatus: NO_BLOCK for not sending the task, BLOCK for waiting, SEND for OK to send
+        """
+        client_name = client_task.client.name
+        task = client_task.task
+        if task.props[_KEY_DYNAMIC_TARGETS]:
+            if task.targets is None:
+                task.targets = []
+            if client_name not in task.targets:
+                task.targets.append(client_name)
+
+        # is this client eligible?
+        if client_name not in task.targets:
+            # this client is not a target
+            return TaskCheckStatus.NO_BLOCK
+
+        client_occurrences = task.targets.count(client_name)
+        finished_targets = task.props[_KEY_FINISHED_TARGETS]
+        send_count = finished_targets.get(client_name, 0)
+        if send_count >= client_occurrences:
+            # already sent enough times to this client
+            return TaskCheckStatus.NO_BLOCK
+
+        # only allow one pending task. Is there a client pending result?
+        pending_client_name = task.props[_KEY_PENDING_CLIENT]
+        task_result_timeout = task.props[_KEY_TASK_RESULT_TIMEOUT]
+        if pending_client_name is not None:
+            # see whether the result has been received
+            pending_task = task.last_client_task_map[pending_client_name]
+            if pending_task.result_received_time is None:
+                # result has not been received
+                # Note: in this case, the pending client and the asking client must not be the
+                # same, because this would be a resend case already taken care of by the controller.
+                if pending_client_name == client_name:
+                    raise RuntimeError("Logic Error: must not be here for client {}".format(client_name))
+
+                # should this client timeout?
+                if task_result_timeout and time.time() - pending_task.task_sent_time > task_result_timeout:
+                    # timeout!
+                    # give up on the pending task and move to the next target
+                    finished_targets[pending_client_name] -= 1
+                    pass
+                else:
+                    # continue to wait
+                    return TaskCheckStatus.BLOCK
+
+        # can send
+        task.props[_KEY_PENDING_CLIENT] = client_name
+        finished_targets[client_name] = send_count + 1
+        return TaskCheckStatus.SEND
+
+    def check_task_exit(self, task: Task) -> Tuple[bool, TaskCompletionStatus]:
+        """Determine whether the task should exit.
+        Args:
+            task (Task): an instance of Task
+
+        Returns:
+            Tuple[bool, TaskCompletionStatus]:
+                first entry in the tuple means whether to exit the task or not.  If it's True, the task should exit.
+                second entry in the tuple indicates the TaskCompletionStatus.
+        """
+        # are we waiting for any client?
+        num_targets = 0 if task.targets is None else len(task.targets)
+        if num_targets == 0:
+            # nothing has been sent
+            return False, TaskCompletionStatus.IGNORED
+
+        # see whether all targets are done
+        finished_targets = task.props[_KEY_FINISHED_TARGETS]
+
+        total_sent = 0
+        for v in finished_targets.values():
+            total_sent += v
+
+        if total_sent < num_targets:
+            return False, TaskCompletionStatus.IGNORED
+
+        for c_t in task.client_tasks:
+            if c_t.result_received_time is None:
+                return False, TaskCompletionStatus.IGNORED
+
+        return True, TaskCompletionStatus.OK
+
+    def check_task_result(self, result: Shareable, client_task: ClientTask, fl_ctx: FLContext):
+        """Check the result received from the client.
+        see whether the client_task is the last one in the task's list
+        If not, then it is a late response and ReservedHeaderKey.REPLY_IS_LATE is
+        set to True in result's header.
+
+        Args:
+            result (Shareable): an instance of Shareable
+            client_task (ClientTask): the task processing state of the client
+            fl_ctx (FLContext): fl context that comes with the task request
+        """
+        task = client_task.task
+        if client_task != task.client_tasks[-1]:
+            result.set_header(key=ReservedHeaderKey.REPLY_IS_LATE, value=True)
diff --git a/nvflare/apis/impl/bcast_manager.py b/nvflare/apis/impl/bcast_manager.py
new file mode 100644
index 0000000000..62309d653e
--- /dev/null
+++ b/nvflare/apis/impl/bcast_manager.py
@@ -0,0 +1,123 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from typing import Tuple
+
+from nvflare.apis.controller_spec import ClientTask, Task, TaskCompletionStatus
+from nvflare.apis.fl_context import FLContext
+
+from .task_manager import TaskCheckStatus, TaskManager
+
+_KEY_MIN_RESPS = "__min_responses"
+_KEY_WAIT_TIME_AFTER_MIN_RESPS = "__wait_time_after_min_received"
+_KEY_MIN_RESPS_RCV_TIME = "__min_resps_received_time"
+
+
+class BcastTaskManager(TaskManager):
+    def __init__(self, task: Task, min_responses: int = 0, wait_time_after_min_received: int = 0):
+        """Task manager for broadcast controller
+
+        Args:
+            task (Task): an instance of Task
+            min_responses (int, optional): the minimum number of responses so this task is considered finished. Defaults to 0.
+            wait_time_after_min_received (int, optional): additional wait time for late clients to contribute their results. Defaults to 0.
+        """
+
+        TaskManager.__init__(self)
+        task.props[_KEY_MIN_RESPS] = min_responses
+        task.props[_KEY_WAIT_TIME_AFTER_MIN_RESPS] = wait_time_after_min_received
+        task.props[_KEY_MIN_RESPS_RCV_TIME] = None
+
+    def check_task_exit(self, task: Task) -> Tuple[bool, TaskCompletionStatus]:
+        """Determine if the task should exit
+
+        Args:
+            task (Task): an instance of Task
+
+        Tuple[bool, TaskCompletionStatus]:
+            first entry in the tuple means whether to exit the task or not.  If it's True, the task should exit.
+            second entry in the tuple indicates the TaskCompletionStatus.
+        """
+        if len(task.client_tasks) == 0:
+            # nothing has been sent - continue to wait
+            return False, TaskCompletionStatus.IGNORED
+
+        clients_responded = 0
+        clients_not_responded = 0
+        for s in task.client_tasks:
+            if s.result_received_time is None:
+                clients_not_responded += 1
+            else:
+                clients_responded += 1
+
+        # if min_responses is 0, need to have all client tasks responded
+        if task.props[_KEY_MIN_RESPS] == 0 and clients_not_responded:
+            return False, TaskCompletionStatus.IGNORED
+
+        # check if minimum responses are received
+        if clients_responded == 0 or clients_responded < task.props[_KEY_MIN_RESPS]:
+            # continue to wait
+            return False, TaskCompletionStatus.IGNORED
+
+        # minimum responses received
+        min_resps_received_time = task.props[_KEY_MIN_RESPS_RCV_TIME]
+        if min_resps_received_time is None:
+            min_resps_received_time = time.time()
+            task.props[_KEY_MIN_RESPS_RCV_TIME] = min_resps_received_time
+
+        # see whether we have waited for long enough
+        if time.time() - min_resps_received_time >= task.props[_KEY_WAIT_TIME_AFTER_MIN_RESPS]:
+            # yes - exit the task
+            return True, TaskCompletionStatus.OK
+        else:
+            # no - continue to wait
+            return False, TaskCompletionStatus.IGNORED
+
+
+class BcastForeverTaskManager(TaskManager):
+    def __init__(self):
+        """Task manager for broadcast controller with forever waiting time"""
+        TaskManager.__init__(self)
+
+    def check_task_send(self, client_task: ClientTask, fl_ctx: FLContext) -> TaskCheckStatus:
+        """Determine whether the task should be sent to the client.
+
+        Args:
+            client_task (ClientTask): the task processing state of the client
+            fl_ctx (FLContext): fl context that comes with the task request
+
+
+        Returns:
+            TaskCheckStatus: NO_BLOCK for not sending the task, SEND for OK to send
+        """
+        # Note: even if the client may have done the task, we may still send it!
+        client_name = client_task.client.name
+        if client_task.task.targets is None or client_name in client_task.task.targets:
+            return TaskCheckStatus.SEND
+        else:
+            return TaskCheckStatus.NO_BLOCK
+
+    def check_task_exit(self, task: Task) -> Tuple[bool, TaskCompletionStatus]:
+        """Determine whether the task should exit.
+
+        Args:
+            task (Task): an instance of Task
+
+        Tuple[bool, TaskCompletionStatus]:
+            first entry in the tuple means whether to exit the task or not.  If it's True, the task should exit.
+            second entry in the tuple indicates the TaskCompletionStatus.
+        """
+        # never exit
+        return False, TaskCompletionStatus.IGNORED
diff --git a/nvflare/apis/impl/controller.py b/nvflare/apis/impl/controller.py
new file mode 100644
index 0000000000..aeb467efa3
--- /dev/null
+++ b/nvflare/apis/impl/controller.py
@@ -0,0 +1,860 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+from abc import ABC
+from threading import Lock
+from typing import List, Optional, Tuple, Union
+
+from nvflare.apis.client import Client
+from nvflare.apis.controller_spec import ClientTask, ControllerSpec, SendOrder, Task, TaskCompletionStatus
+from nvflare.apis.fl_constant import FLContextKey, ReservedTopic
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.fl_exception import WorkflowError
+from nvflare.apis.responder import Responder
+from nvflare.apis.server_engine_spec import ServerEngineSpec
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.widgets.info_collector import GroupInfoCollector, InfoCollector
+
+from .any_relay_manager import AnyRelayTaskManager
+from .bcast_manager import BcastForeverTaskManager, BcastTaskManager
+from .send_manager import SendTaskManager
+from .seq_relay_manager import SequentialRelayTaskManager
+from .task_manager import TaskCheckStatus, TaskManager
+
+_TASK_KEY_ENGINE = "___engine"
+_TASK_KEY_MANAGER = "___mgr"
+_TASK_KEY_DONE = "___done"
+
+
+def _check_positive_int(name, value):
+    if not isinstance(value, int):
+        raise TypeError(f"{name} must be an instance of int.")
+    if value < 0:
+        raise ValueError(f"{name} must >= 0.")
+
+
+def _check_inputs(task: Task, fl_ctx: FLContext, targets: Union[List[Client], List[str], None]):
+    if not isinstance(task, Task):
+        raise TypeError("task must be an instance of Task.")
+
+    if not isinstance(fl_ctx, FLContext):
+        raise TypeError("fl_ctx must be an instance of FLContext.")
+
+    if targets is not None:
+        if not isinstance(targets, list):
+            raise TypeError("targets must be a list of Client or string.")
+
+        if not all([isinstance(t, (Client, str)) for t in targets]):
+            raise TypeError("targets must be a list of Client or string.")
+
+
+class Controller(Responder, ControllerSpec, ABC):
+    def __init__(self, task_check_period=0.5):
+        """Manage life cycles of tasks and their destinations
+
+        Args:
+            task_check_period (float, optional): interval for checking status of tasks. Defaults to 0.5.
+        """
+        Responder.__init__(self)
+        self._engine = None
+        self._tasks = []  # list of standing tasks
+        self._client_task_map = {}  # client_task_id => client_task
+        self._all_done = False
+        self._task_lock = Lock()
+        self._task_monitor = threading.Thread(target=self._monitor_tasks, args=())
+        self._task_check_period = task_check_period
+
+    def initialize_run(self, fl_ctx: FLContext):
+        """Called by runners to initailze controller with information in fl_ctx
+
+        Note: Controller subclasses must not overwrite this method.
+
+        Args:
+            fl_ctx (FLContext): FLContext information
+        """
+
+        self._engine = fl_ctx.get_engine()
+        self.start_controller(fl_ctx)
+        self._task_monitor.start()
+
+    def _try_again(self) -> Tuple[str, str, Shareable]:
+        # TODO: how to tell client no shareable available now?
+        return "", "", None
+
+    def _set_stats(self, fl_ctx: FLContext):
+        """Called to set stats into InfoCollector
+
+        Args:
+            fl_ctx (FLContext): info collector is retrieved from fl_ctx with InfoCollector.CTX_KEY_STATS_COLLECTOR key
+        """
+        collector = fl_ctx.get_prop(InfoCollector.CTX_KEY_STATS_COLLECTOR, None)
+        if collector:
+            assert isinstance(collector, GroupInfoCollector)
+            collector.set_info(
+                group_name=self._name,
+                info={
+                    "tasks": {t.name: [ct.client.name for ct in t.client_tasks] for t in self._tasks},
+                },
+            )
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        """Called when events are fired
+
+        Args:
+            event_type (str): all event types, including AppEventType and EventType
+            fl_ctx (FLContext): FLContext information with current event type
+        """
+        if event_type == InfoCollector.EVENT_TYPE_GET_STATS:
+            self._set_stats(fl_ctx)
+
+    def process_task_request(self, client: Client, fl_ctx: FLContext) -> Tuple[str, str, Shareable]:
+        """Called by runner when a client asks for a task
+
+        Note: this is called in a separate thread.
+
+        Args:
+            client (Client): The record of one client requesting tasks
+            fl_ctx (FLContext): The FLContext associated with this request
+
+        Raises:
+            TypeError: when client is not an instance of Client
+            TypeError: when fl_ctx is not an instance of FLContext
+            TypeError: when any standing task containing an invalid client_task
+
+        Returns:
+            Tuple[str, str, Shareable]: task_name, an id for the client_task, and the data for this requst
+        """
+        if not isinstance(client, Client):
+            raise TypeError("client must be an instance of Client.")
+        if not isinstance(fl_ctx, FLContext):
+            raise TypeError("fl_ctx must be an instance of FLContext.")
+
+        client_task_to_send = None
+        with self._task_lock:
+            self.logger.debug(f"{self._tasks=}")
+            for task in self._tasks:
+                assert isinstance(task, Task)
+
+                if task.completion_status is not None:
+                    # this task is finished (and waiting for the monitor to exit it)
+                    continue
+
+                # do we need to send this task to this client?
+                # note: the task could be sent to a client multiple times (e.g. in relay)
+                # we only check the last ClientTask sent to the client!
+                client_task_to_check = task.last_client_task_map.get(client.name, None)
+                self.logger.debug(f"task to check {client_task_to_check=}")
+                resend_task = False
+
+                if client_task_to_check is not None:
+                    # this client has been sent the task already.
+                    # do we need to send again?
+                    if not isinstance(client_task_to_check, ClientTask):
+                        raise TypeError(f"{client_task_to_check=} must be an instance of ClientTask.")
+                    if client_task_to_check.result_received_time is None:
+                        # we have not received result
+                        # the client must be working on this task, but apparently something is wrong
+                        # we resend the task!
+                        resend_task = True
+                        client_task_to_send = client_task_to_check
+                        fl_ctx.set_prop(FLContextKey.IS_CLIENT_TASK_RESEND, True, sticky=False)
+
+                if not resend_task:
+                    # check with the task manager whether to send
+                    manager = task.props[_TASK_KEY_MANAGER]
+                    if client_task_to_check is None:
+                        client_task_to_check = ClientTask(task=task, client=client)
+                    check_status = manager.check_task_send(client_task_to_check, fl_ctx)
+                    self.logger.debug(f"checking client task {client_task_to_check} {client_task_to_check.client.name}")
+                    self.logger.debug(f"Check task send get {check_status=}")
+                    if check_status == TaskCheckStatus.BLOCK:
+                        # TODO: make sure engine understand the return value
+                        # do not send this task, and do not check other tasks
+                        return self._try_again()
+                    elif check_status == TaskCheckStatus.NO_BLOCK:
+                        # do not send this task, but continue to check next task
+                        continue
+                    else:
+                        # go ahead to send the task!
+                        # only need to remember the client_task if task is sent to it!
+                        client_task_to_send = ClientTask(client, task)
+                        self.logger.debug(f"mapping {client.name} to {client_task_to_send}")
+                        task.last_client_task_map[client.name] = client_task_to_send
+                        task.client_tasks.append(client_task_to_send)
+                        self.logger.debug(f"{task.client_tasks=}")
+                        self.logger.debug(f"Adding {client_task_to_send.id=}")
+                        self._client_task_map[client_task_to_send.id] = client_task_to_send
+                        break
+
+        # NOTE: move task sending process outside of the lock
+        # This is to minimize the locking time.
+        # Also to avoid potential dead lock - the CB could schedule another task, which requires lock
+        self.logger.debug(f"Determining based on {client_task_to_send=}")
+        if client_task_to_send is None:
+            # no task available for this client
+            return self._try_again()
+
+        # now try to send
+        can_send_task = True
+        task = client_task_to_send.task
+        with task.cb_lock:
+            # Note: must guarantee the after_task_sent_cb is always called
+            # regardless whether the task is sent successfully!
+            # This is so that the app could clear up things in after_task_sent_cb.
+            if task.before_task_sent_cb is not None:
+                try:
+                    task.before_task_sent_cb(client_task=client_task_to_send, fl_ctx=fl_ctx)
+                except WorkflowError as ex:
+                    self._engine.ask_to_stop()
+                    self.log_exception(
+                        fl_ctx,
+                        "processing error in before_task_sent_cb on task {}:{}".format(
+                            client_task_to_send.task.name, client_task_to_send.id
+                        ),
+                    )
+                    task.completion_status = TaskCompletionStatus.ERROR
+                    task.exception = ex
+                except BaseException as ex:
+                    # this task cannot proceed any more
+                    task.completion_status = TaskCompletionStatus.ERROR
+                    task.exception = ex
+
+            self.logger.debug(f"before_task_sent_cb done on {client_task_to_send=}")
+
+            if task.completion_status is not None:
+                can_send_task = False
+
+            # remember the task name and data to be sent to the client
+            # since task.data could be reset by the after_task_sent_cb!
+            task_name = task.name
+            task_data = task.data
+
+            if task.after_task_sent_cb is not None:
+                try:
+                    task.after_task_sent_cb(client_task=client_task_to_send, fl_ctx=fl_ctx)
+                except WorkflowError as ex:
+                    self._engine.ask_to_stop()
+                    self.log_exception(
+                        fl_ctx,
+                        "processing error in after_task_sent_cb on task {}:{}".format(
+                            client_task_to_send.task.name, client_task_to_send.id
+                        ),
+                    )
+                    task.completion_status = TaskCompletionStatus.ERROR
+                    task.exception = ex
+                except BaseException as ex:
+                    # this task cannot proceed any more
+                    task.completion_status = TaskCompletionStatus.ERROR
+                    task.exception = ex
+
+            if task.completion_status is not None:
+                # NOTE: the CB could cancel the task!
+                # tell the client to try again
+                can_send_task = False
+
+            if not can_send_task:
+                return self._try_again()
+
+            self.logger.debug(f"after_task_sent_cb done on {client_task_to_send=}")
+
+            client_task_to_send.task_sent_time = time.time()
+            client_task_to_send.task_send_count += 1
+            return task_name, client_task_to_send.id, task_data
+
+    def handle_exception(self, task_id: str, fl_ctx: FLContext) -> None:
+        """Called to cancel one task as its client_task is causing exception at upper level.
+
+        Args:
+            task_id (str): an id to the failing client_task
+            fl_ctx (FLContext): FLContext associated with this client_task
+        """
+        with self._task_lock:
+            # task_id is the uuid associated with the client_task
+            self.logger.debug(f"Handle exception on {task_id=}")
+            client_task = self._client_task_map.get(task_id, None)
+
+        self.logger.debug(f"{client_task=}")
+        if client_task is None:
+            # cannot find a standing task on the exception
+            return
+
+        task = client_task.task
+        self.cancel_task(task=task, fl_ctx=fl_ctx)
+        self.log_error(fl_ctx, "task {} is cancelled due to exception".format(task.name))
+
+    def process_submission(self, client: Client, task_name: str, task_id: str, result: Shareable, fl_ctx: FLContext):
+        """Called to process a submission from one client.
+
+        Note: this method is called by a separate thread.
+
+        Args:
+            client (Client): the client that submitted this task
+            task_name (str): the task name associated this submission
+            task_id (str): the id associated with the client_task
+            result (Shareable): the actual submitted data from the client
+            fl_ctx (FLContext): the FLContext associated with this submission
+
+        Raises:
+            TypeError: when client is not an instance of Client
+            TypeError: when fl_ctx is not an instance of FLContext
+            TypeError: when result is not an instance of Shareable
+            ValueError: task_name is not found in the client_task
+        """
+        if not isinstance(client, Client):
+            raise TypeError("client must be an instance of Client.")
+        if not isinstance(fl_ctx, FLContext):
+            raise TypeError("fl_ctx must be an instance of FLContext.")
+        if not isinstance(result, Shareable):
+            raise TypeError("result must be an instance of Shareable.")
+
+        with self._task_lock:
+            # task_id is the uuid associated with the client_task
+            self.logger.debug(f"Submission {task_id=}")
+            client_task = self._client_task_map.get(task_id, None)
+
+        self.logger.debug(f"{client_task=}")
+        if client_task is None:
+            # cannot find a standing task for the submission
+            self.log_info(fl_ctx, "no standing task found for {}:{}".format(task_name, task_id))
+            self.process_result_of_unknown_task(client, task_name, task_id, result, fl_ctx)
+            return
+
+        assert isinstance(client_task, ClientTask)
+        task = client_task.task
+        with task.cb_lock:
+            if task.name != task_name:
+                raise ValueError("client specified task name {} doesn't match {}".format(task_name, task.name))
+
+            if task.completion_status is not None:
+                # the task is already finished - drop the result
+                self.log_info(fl_ctx, "task is already finished - submission dropped")
+                return
+
+            # do client task CB processing outside of the lock!
+            # this is because the CB could schedule another task, which requires the lock!
+            self.logger.debug(f"Submission {result=}")
+            self.logger.debug(f"{client_task.client.name=}")
+            client_task.result = result
+            client_task.result_received_time = time.time()
+
+            manager = task.props[_TASK_KEY_MANAGER]
+            assert isinstance(manager, TaskManager)
+            manager.check_task_result(result, client_task, fl_ctx)
+
+            if task.result_received_cb is not None:
+                try:
+                    self.log_info(fl_ctx, "invoking result_received_cb ...")
+                    task.result_received_cb(client_task=client_task, fl_ctx=fl_ctx)
+                except WorkflowError as ex:
+                    self._engine.ask_to_stop()
+                    self.log_exception(
+                        fl_ctx,
+                        "processing error in result_received_cb on task {}:{}".format(task_name, task_id),
+                    )
+                    task.completion_status = TaskCompletionStatus.ERROR
+                    task.exception = ex
+                    return
+                except BaseException as ex:
+                    # this task cannot proceed any more
+                    self.log_exception(
+                        fl_ctx,
+                        "processing error in result_received_cb on task {}:{}".format(task_name, task_id),
+                    )
+                    task.completion_status = TaskCompletionStatus.ERROR
+                    task.exception = ex
+                    return
+            else:
+                self.log_info(fl_ctx, "no result_received_cb")
+
+    def _schedule_task(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        manager: TaskManager,
+        targets: Union[List[Client], List[str], None],
+        allow_dup_targets: bool = False,
+    ):
+        if task.schedule_time is not None:
+            # this task was scheduled before
+            # we do not allow a task object to be reused!
+            self.logger.debug(f"{task.schedule_time=}")
+            raise ValueError("Task was already used. Please create a new task object.")
+
+        task.targets = targets
+        if targets is not None:
+            target_names = list()
+            if not isinstance(targets, list):
+                raise ValueError("task targets must be a list.")
+            for t in targets:
+                if isinstance(t, str):
+                    name = t
+                elif isinstance(t, Client):
+                    name = t.name
+                else:
+                    raise ValueError("element in targets must be string or Client type.")
+
+                if allow_dup_targets or (name not in target_names):
+                    target_names.append(name)
+            task.targets = target_names
+
+        task.props[_TASK_KEY_MANAGER] = manager
+        task.props[_TASK_KEY_ENGINE] = fl_ctx.get_engine()
+        task.is_standing = True
+        task.schedule_time = time.time()
+
+        with self._task_lock:
+            self.logger.debug(f"{task=} added")
+            self._tasks.append(task)
+            self.log_info(fl_ctx, "scheduled task {}".format(task.name))
+
+    def broadcast(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        min_responses: int = 1,
+        wait_time_after_min_received: int = 0,
+    ):
+        """Schedule a broadcast task.  This is a non-blocking call.
+
+        The task is scheduled into a task list.  Clients can request tasks and controller will dispatch the task to eligible clients.
+
+        Args:
+            task (Task): the task to be scheduled
+            fl_ctx (FLContext): FLContext associated with this task
+            targets (Union[List[Client], List[str], None], optional): the list of eligible clients or client names or None (all clients). Defaults to None.
+            min_responses (int, optional): the condition to mark this task as completed because enough clients respond with submission. Defaults to 1.
+            wait_time_after_min_received (int, optional): a grace period for late clients to contribute their submission.  0 means no grace period.
+            Submission of late clients in the grace period are still collected as valid submission. Defaults to 0.
+
+        Raises:
+            ValueError: min_responses is greater than the length of targets since this condition will make the task, if allowed to be scheduled, never exit.
+        """
+        _check_inputs(task=task, fl_ctx=fl_ctx, targets=targets)
+        _check_positive_int("min_responses", min_responses)
+        _check_positive_int("wait_time_after_min_received", wait_time_after_min_received)
+        if targets and min_responses > len(targets):
+            raise ValueError("min_responses must be less than length of targets.")
+
+        manager = BcastTaskManager(
+            task=task, min_responses=min_responses, wait_time_after_min_received=wait_time_after_min_received
+        )
+        self._schedule_task(task=task, fl_ctx=fl_ctx, manager=manager, targets=targets)
+
+    def broadcast_and_wait(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        min_responses: int = 1,
+        wait_time_after_min_received: int = 0,
+        abort_signal: Optional[Signal] = None,
+    ):
+        """Schedule a broadcast task.  This is a blocking call.
+
+        The task is scheduled into a task list.  Clients can request tasks and controller will dispatch the task to eligible clients.
+
+        Args:
+            task (Task): the task to be scheduled
+            fl_ctx (FLContext): FLContext associated with this task
+            targets (Union[List[Client], List[str], None], optional): the list of eligible clients or client names or None (all clients). Defaults to None.
+            min_responses (int, optional): the condition to mark this task as completed because enough clients respond with submission. Defaults to 1.
+            wait_time_after_min_received (int, optional): a grace period for late clients to contribute their submission.  0 means no grace period.
+            Submission of late clients in the grace period are still collected as valid submission. Defaults to 0.
+            abort_signal (Optional[Signal], optional): as this is a blocking call, this abort_signal informs this method to return. Defaults to None.
+        """
+        self.broadcast(
+            task=task,
+            fl_ctx=fl_ctx,
+            targets=targets,
+            min_responses=min_responses,
+            wait_time_after_min_received=wait_time_after_min_received,
+        )
+        self._wait_for_task(task, abort_signal)
+
+    def broadcast_forever(self, task: Task, fl_ctx: FLContext, targets: Union[List[Client], List[str], None] = None):
+        """Schedule a broadcast task.  This is a non-blocking call.
+
+        The task is scheduled into a task list.  Clients can request tasks and controller will dispatch the task to eligible clients.
+        This broadcast will not end.
+
+        Args:
+            task (Task): the task to be scheduled
+            fl_ctx (FLContext): FLContext associated with this task
+            targets (Union[List[Client], List[str], None], optional): the list of eligible clients or client names or None (all clients). Defaults to None.
+        """
+        _check_inputs(task=task, fl_ctx=fl_ctx, targets=targets)
+        manager = BcastForeverTaskManager()
+        self._schedule_task(task=task, fl_ctx=fl_ctx, manager=manager, targets=targets)
+
+    def send(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        send_order: SendOrder = SendOrder.SEQUENTIAL,
+        task_assignment_timeout: int = 0,
+    ):
+        """Schedule a single task to targets.  This is a non-blocking call.
+
+        The task is scheduled into a task list.  Clients can request tasks and controller will dispatch the task to eligible clients based on the send_order.
+
+        Args:
+            task (Task): the task to be scheduled
+            fl_ctx (FLContext): FLContext associated with this task
+            targets (Union[List[Client], List[str], None], optional): the list of eligible clients or client names or None (all clients). Defaults to None.
+            send_order (SendOrder, optional): the order for clients to become eligible.  SEQUENTIAL means the order in targets is enforced.  ANY means
+            clients in targets and haven't received task are eligible for task. Defaults to SendOrder.SEQUENTIAL.
+            task_assignment_timeout (int, optional): how long to wait for one client to pick the task. Defaults to 0.
+
+        Raises:
+            ValueError: when task_assignment_timeout is greater than task's timeout.
+            TypeError: send_order is not defined in SendOrder
+            ValueError: targets is None or an empty list
+        """
+        _check_inputs(
+            task=task,
+            fl_ctx=fl_ctx,
+            targets=targets,
+        )
+        _check_positive_int("task_assignment_timeout", task_assignment_timeout)
+        if task.timeout and task_assignment_timeout and task_assignment_timeout > task.timeout:
+            raise ValueError("task_assignment_timeout need to be less than or equal to task.timeout.")
+        if not isinstance(send_order, SendOrder):
+            raise TypeError("send_order must be in Enum SendOrder.")
+
+        # targets must be provided!
+        if targets is None or len(targets) == 0:
+            raise ValueError("Targets must be provided for send.")
+
+        manager = SendTaskManager(task, send_order, task_assignment_timeout)
+        self._schedule_task(
+            task=task,
+            fl_ctx=fl_ctx,
+            manager=manager,
+            targets=targets,
+        )
+
+    def send_and_wait(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        send_order: SendOrder = SendOrder.SEQUENTIAL,
+        task_assignment_timeout: int = 0,
+        abort_signal: Signal = None,
+    ):
+        """Schedule a single task to targets.  This is a blocking call.
+
+        The task is scheduled into a task list.  Clients can request tasks and controller will dispatch the task to eligible clients based on the send_order.
+
+        Args:
+            task (Task): the task to be scheduled
+            fl_ctx (FLContext): FLContext associated with this task
+            targets (Union[List[Client], List[str], None], optional): the list of eligible clients or client names or None (all clients). Defaults to None.
+            send_order (SendOrder, optional): the order for clients to become eligible.  SEQUENTIAL means the order in targets is enforced.  ANY means
+            clients in targets and haven't received task are eligible for task. Defaults to SendOrder.SEQUENTIAL.
+            task_assignment_timeout (int, optional): how long to wait for one client to pick the task. Defaults to 0.
+            abort_signal (Optional[Signal], optional): as this is a blocking call, this abort_signal informs this method to return. Defaults to None.
+
+        """
+        self.send(
+            task=task,
+            fl_ctx=fl_ctx,
+            targets=targets,
+            send_order=send_order,
+            task_assignment_timeout=task_assignment_timeout,
+        )
+        self._wait_for_task(task, abort_signal)
+
+    def get_num_standing_tasks(self) -> int:
+        """Get the number of tasks that are currently standing.
+
+        Returns:
+            int: length of the list of standing tasks
+        """
+        return len(self._tasks)
+
+    def cancel_task(
+        self, task: Task, completion_status=TaskCompletionStatus.CANCELLED, fl_ctx: Optional[FLContext] = None
+    ):
+        """Cancel the specified task.
+
+        Change the task completion_status, which will inform task monitor to clean up this task
+        NOTE: we only mark the task as completed and leave it to the task monitor to clean up
+        This is to avoid potential dead lock of task_lock
+
+        Args:
+            task (Task): the task to be cancelled
+            completion_status ([type], optional): the completion status for this cancellation. Defaults to TaskCompletionStatus.CANCELLED.
+            fl_ctx (Optional[FLContext], optional): FLContext associated with this cancellation. Defaults to None.
+        """
+        task.completion_status = completion_status
+
+    def cancel_all_tasks(self, completion_status=TaskCompletionStatus.CANCELLED, fl_ctx: Optional[FLContext] = None):
+        """Cancel all standing tasks in this controller.
+
+        Args:
+            completion_status ([type], optional): the completion status for this cancellation. Defaults to TaskCompletionStatus.CANCELLED.
+            fl_ctx (Optional[FLContext], optional): FLContext associated with this cancellation. Defaults to None.
+        """
+        with self._task_lock:
+            for t in self._tasks:
+                t.completion_status = completion_status
+
+    def abort_task(self, task, fl_ctx: FLContext):
+        """Ask all clients to abort the execution of the specified task.
+
+        Args:
+            task ([type]): the task to be aborted
+            fl_ctx (FLContext): FLContext associated with this action
+        """
+        self.log_info(fl_ctx, "asked all clients to abort task {}".format(task.name))
+        self._end_task([task.name], fl_ctx)
+
+    def _end_task(self, task_names, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        assert isinstance(engine, ServerEngineSpec)
+        request = Shareable()
+        request["task_names"] = task_names
+        engine.send_aux_request(targets=None, topic=ReservedTopic.ABORT_ASK, request=request, timeout=0, fl_ctx=fl_ctx)
+
+    def abort_all_tasks(self, fl_ctx: FLContext):
+        """Ask clients to abort the execution of all tasks.
+        NOTE: the server should send a notification to all clients, regardless of whether the server
+        has any standing tasks.
+
+        Args:
+            fl_ctx (FLContext): FLContext associated with this action
+        """
+        self._end_task([], fl_ctx)
+
+    def finalize_run(self, fl_ctx: FLContext):
+        """Do cleanup of the coordinator implementation.
+        NOTE: subclass controllers should not overwrite finalize_run.
+
+        Args:
+            fl_ctx (FLContext): FLContext associated with this action
+        """
+        self.cancel_all_tasks()  # unconditionally cancel all tasks
+        self._all_done = True
+        try:
+            if self._task_monitor.is_alive():
+                self._task_monitor.join()
+        except RuntimeError:
+            self.log_debug(fl_ctx, "unable to join monitor thread (not started?)")
+        self.stop_controller(fl_ctx)
+
+    def relay(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        send_order: SendOrder = SendOrder.SEQUENTIAL,
+        task_assignment_timeout: int = 0,
+        task_result_timeout: int = 0,
+        dynamic_targets: bool = True,
+    ):
+        """Schedule a single task to targets in one-after-another style.  This is a non-blocking call.
+
+        The task is scheduled into a task list.  Clients can request tasks and controller will dispatch the task to eligible clients based on the send_order.
+
+        Args:
+            task (Task): the task to be scheduled
+            fl_ctx (FLContext): FLContext associated with this task
+            targets (Union[List[Client], List[str], None], optional): the list of eligible clients or client names or None (all clients). Defaults to None.
+            send_order (SendOrder, optional): the order for clients to become eligible.
+              SEQUENTIAL means the order in targets is enforced.
+              ANY means any clients that are inside the targets and haven't received the task are eligible. Defaults to SendOrder.SEQUENTIAL.
+            task_assignment_timeout (int, optional): how long to wait for one client to pick the task. Defaults to 0.
+            task_result_timeout (int, optional): how long to wait for current working client to reply its result. Defaults to 0.
+            dynamic_targets (bool, optional): allow clients not in targets to join at the end of targets list. Defaults to True.
+
+        Raises:
+            ValueError: when task_assignment_timeout is greater than task's timeout
+            ValueError: when task_result_timeout is greater than task's timeout
+            TypeError: send_order is not defined in SendOrder
+            TypeError: when dynamic_targets is not a boolean variable
+            ValueError: targets is None or an empty list but dynamic_targets is False
+        """
+        _check_inputs(
+            task=task,
+            fl_ctx=fl_ctx,
+            targets=targets,
+        )
+        _check_positive_int("task_assignment_timeout", task_assignment_timeout)
+        _check_positive_int("task_result_timeout", task_result_timeout)
+        if task.timeout and task_assignment_timeout and task_assignment_timeout > task.timeout:
+            raise ValueError("task_assignment_timeout need to be less than or equal to task.timeout.")
+        if task.timeout and task_result_timeout and task_result_timeout > task.timeout:
+            raise ValueError("task_result_timeout need to be less than or equal to task.timeout.")
+        if not isinstance(send_order, SendOrder):
+            raise TypeError("send_order must be in Enum SendOrder.")
+        if not isinstance(dynamic_targets, bool):
+            raise TypeError("dynamic_targets must be an instance of bool.")
+        if targets is None and dynamic_targets is False:
+            raise ValueError("Need to provide targets when dynamic_targets is set to False.")
+
+        if send_order == SendOrder.SEQUENTIAL:
+            manager = SequentialRelayTaskManager(
+                task=task,
+                task_assignment_timeout=task_assignment_timeout,
+                task_result_timeout=task_result_timeout,
+                dynamic_targets=dynamic_targets,
+            )
+        else:
+            manager = AnyRelayTaskManager(
+                task=task, task_result_timeout=task_result_timeout, dynamic_targets=dynamic_targets
+            )
+
+        self._schedule_task(
+            task=task,
+            fl_ctx=fl_ctx,
+            manager=manager,
+            targets=targets,
+            allow_dup_targets=True,
+        )
+
+    def relay_and_wait(
+        self,
+        task: Task,
+        fl_ctx: FLContext,
+        targets: Union[List[Client], List[str], None] = None,
+        send_order=SendOrder.SEQUENTIAL,
+        task_assignment_timeout: int = 0,
+        task_result_timeout: int = 0,
+        dynamic_targets: bool = True,
+        abort_signal: Optional[Signal] = None,
+    ):
+        """Schedule a single task to targets in one-after-another style.  This is a blocking call.
+
+        The task is scheduled into a task list.  Clients can request tasks and controller will dispatch the task to eligible clients based on the send_order.
+
+        Args:
+            task (Task): the task to be scheduled
+            fl_ctx (FLContext): FLContext associated with this task
+            targets (Union[List[Client], List[str], None], optional): the list of eligible clients or client names or None (all clients). Defaults to None.
+            send_order (SendOrder, optional): the order for clients to become eligible.  SEQUENTIAL means the order in targets is enforced.  ANY means
+            clients in targets and haven't received task are eligible for task. Defaults to SendOrder.SEQUENTIAL.
+            task_assignment_timeout (int, optional): how long to wait for one client to pick the task. Defaults to 0.
+            task_result_timeout (int, optional): how long to wait for current working client to reply its result. Defaults to 0.
+            dynamic_targets (bool, optional): allow clients not in targets to join at the end of targets list. Defaults to True.
+            abort_signal (Optional[Signal], optional): as this is a blocking call, this abort_signal informs this method to return. Defaults to None.
+        """
+        self.relay(
+            task=task,
+            fl_ctx=fl_ctx,
+            targets=targets,
+            send_order=send_order,
+            task_assignment_timeout=task_assignment_timeout,
+            task_result_timeout=task_result_timeout,
+            dynamic_targets=dynamic_targets,
+        )
+        self._wait_for_task(task, abort_signal)
+
+    def _monitor_tasks(self):
+        while not self._all_done:
+            self._check_tasks()
+            time.sleep(self._task_check_period)
+
+    def _check_tasks(self):
+        exit_tasks = []
+        with self._task_lock:
+            for task in self._tasks:
+                if task.completion_status is not None:
+                    exit_tasks.append(task)
+                    continue
+
+                # check the task-specific exit condition
+                manager = task.props[_TASK_KEY_MANAGER]
+                if manager is not None:
+                    if not isinstance(manager, TaskManager):
+                        raise TypeError("manager in task must be an instance of TaskManager.")
+                    should_exit, exit_status = manager.check_task_exit(task)
+                    self.logger.debug(f"{should_exit=}, {exit_status=}")
+                    if should_exit:
+                        task.completion_status = exit_status
+                        exit_tasks.append(task)
+                        continue
+
+                # check if task timeout
+                if task.timeout and time.time() - task.schedule_time >= task.timeout:
+                    task.completion_status = TaskCompletionStatus.TIMEOUT
+                    exit_tasks.append(task)
+                    continue
+
+            for exit_task in exit_tasks:
+                exit_task.is_standing = False
+                self.logger.debug(f"Removing {exit_task=} for {exit_task.completion_status=}")
+                self._tasks.remove(exit_task)
+                for client_task in exit_task.client_tasks:
+                    self.logger.debug(f"Removing {client_task.id=}")
+                    self._client_task_map.pop(client_task.id)
+
+        # do the task exit processing outside of the lock to minimize the locking time!
+        # also to avoid potential dead lock since the CB could schedule another task
+        if len(exit_tasks) <= 0:
+            return
+
+        with self._engine.new_context() as fl_ctx:
+            for exit_task in exit_tasks:
+                assert isinstance(exit_task, Task)
+                with exit_task.cb_lock:
+                    self.log_info(
+                        fl_ctx, "task {} exit with status {}".format(exit_task.name, exit_task.completion_status)
+                    )
+
+                    if exit_task.task_done_cb is not None:
+                        try:
+                            exit_task.task_done_cb(task=exit_task, fl_ctx=fl_ctx)
+                        except WorkflowError as ex:
+                            self._engine.ask_to_stop()
+                            self.log_exception(
+                                fl_ctx,
+                                "processing task_done_cb() error on task {}".format(exit_task.name),
+                            )
+                            task.completion_status = TaskCompletionStatus.ERROR
+                            task.exception = ex
+                        except BaseException as ex:
+                            self.log_exception(fl_ctx, f"Exception {ex} raised in task_done_cb on task={exit_task}")
+                            exit_task.completion_status = TaskCompletionStatus.ERROR
+                            exit_task.exception = ex
+
+    @staticmethod
+    def _process_finished_task(task, func):
+        def wrap(*args, **kwargs):
+            if func:
+                func(*args, **kwargs)
+            task.props[_TASK_KEY_DONE] = True
+
+        return wrap
+
+    def _wait_for_task(self, task: Task, abort_signal: Signal):
+        task.props[_TASK_KEY_DONE] = False
+        task.task_done_cb = self._process_finished_task(task=task, func=task.task_done_cb)
+        while True:
+            if task.completion_status is not None:
+                break
+
+            if abort_signal and abort_signal.triggered:
+                self.cancel_task(task, fl_ctx=None, completion_status=TaskCompletionStatus.ABORTED)
+                break
+
+            task_done = task.props[_TASK_KEY_DONE]
+            if task_done:
+                break
+            time.sleep(self._task_check_period)
diff --git a/nvflare/apis/impl/send_manager.py b/nvflare/apis/impl/send_manager.py
new file mode 100644
index 0000000000..60ef3b5dd8
--- /dev/null
+++ b/nvflare/apis/impl/send_manager.py
@@ -0,0 +1,111 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from typing import Tuple
+
+from nvflare.apis.controller_spec import ClientTask, SendOrder, Task, TaskCompletionStatus
+from nvflare.apis.fl_context import FLContext
+
+from .task_manager import TaskCheckStatus, TaskManager
+
+_KEY_ORDER = "__order"
+_KEY_TASK_ASSIGN_TIMEOUT = "__task_assignment_timeout"
+
+
+class SendTaskManager(TaskManager):
+    def __init__(self, task: Task, send_order: SendOrder, task_assignment_timeout):
+        """Task manager for send controller
+
+        Args:
+            task (Task): an instance of Task
+            send_order (SendOrder): the order of clients to receive task
+            task_assignment_timeout (int): timeout value on a client requesting its task
+        """
+        TaskManager.__init__(self)
+        if task_assignment_timeout is None or task_assignment_timeout <= 0:
+            task_assignment_timeout = 0
+        task.props[_KEY_ORDER] = send_order
+        task.props[_KEY_TASK_ASSIGN_TIMEOUT] = task_assignment_timeout
+
+    def check_task_send(self, client_task: ClientTask, fl_ctx: FLContext) -> TaskCheckStatus:
+        """Determine whether the task should be sent to the client.
+
+        Args:
+            client_task (ClientTask): the task processing state of the client
+            fl_ctx (FLContext): fl context that comes with the task request
+
+        Returns:
+            TaskCheckStatus: NO_BLOCK for not sending the task, BLOCK for waiting, SEND for OK to send
+        """
+        task = client_task.task
+        if len(task.client_tasks) > 0:  # already sent to one client
+            if client_task.task_sent_time is not None:  # the task was sent to this client!
+                if client_task.result_received_time is not None:
+                    # the task result was already received
+                    # this task is actually done - waiting to end by the monitor
+                    return TaskCheckStatus.NO_BLOCK
+                else:
+                    return TaskCheckStatus.SEND
+            else:  # the task was sent to someone else
+                return TaskCheckStatus.NO_BLOCK
+
+        # in SEQUENTIAL mode - targets must be explicitly specified
+        # is this client eligible?
+        try:
+            client_idx = task.targets.index(client_task.client.name)
+        except ValueError:
+            client_idx = -1
+
+        if client_idx < 0:
+            # this client is not a target
+            return TaskCheckStatus.NO_BLOCK
+
+        if task.props[_KEY_ORDER] == SendOrder.ANY:
+            return TaskCheckStatus.SEND
+
+        task_assignment_timeout = task.props[_KEY_TASK_ASSIGN_TIMEOUT]
+        if task_assignment_timeout == 0:
+            # no client timeout - can only send to the first target
+            eligible_client_idx = 0
+        else:
+            elapsed = time.time() - task.create_time
+            eligible_client_idx = int(elapsed / task_assignment_timeout)
+
+        if client_idx <= eligible_client_idx:
+            return TaskCheckStatus.SEND
+        else:
+            # this client is currently not eligible but could be later
+            # since this client is involved in the task, we need to wait until this task is resolved!
+            return TaskCheckStatus.BLOCK
+
+    def check_task_exit(self, task: Task) -> Tuple[bool, TaskCompletionStatus]:
+        """Determine whether the task should exit.
+        Args:
+            task (Task): an instance of Task
+
+        Tuple[bool, TaskCompletionStatus]:
+            first entry in the tuple means whether to exit the task or not.  If it's True, the task should exit.
+            second entry in the tuple indicates the TaskCompletionStatus.
+        """
+        if len(task.client_tasks) > 0:
+            # there should be only a single item in the task's client status list
+            # because only a single client is sent the task!
+            for s in task.client_tasks:
+                if s.result_received_time is not None:
+                    # this task is done!
+                    return True, TaskCompletionStatus.OK
+
+        # no one is working on this task yet or the task is not done
+        return False, TaskCompletionStatus.IGNORED
diff --git a/nvflare/apis/impl/seq_relay_manager.py b/nvflare/apis/impl/seq_relay_manager.py
new file mode 100644
index 0000000000..7ad01085e2
--- /dev/null
+++ b/nvflare/apis/impl/seq_relay_manager.py
@@ -0,0 +1,198 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from typing import Tuple
+
+from nvflare.apis.controller_spec import ClientTask, Task, TaskCompletionStatus
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import ReservedHeaderKey, Shareable
+
+from .task_manager import TaskCheckStatus, TaskManager
+
+_KEY_DYNAMIC_TARGETS = "__dynamic_targets"
+_KEY_TASK_ASSIGN_TIMEOUT = "__task_assignment_timeout"
+_KEY_TASK_RESULT_TIMEOUT = "__task_result_timeout"
+_KEY_LAST_SEND_IDX = "__last_send_idx"
+_PENDING_CLIENT_TASK = "__pending_client_task"
+
+
+class SequentialRelayTaskManager(TaskManager):
+    def __init__(self, task: Task, task_assignment_timeout, task_result_timeout, dynamic_targets: bool):
+        """Task manager for relay controller on SendOrder.SEQUENTIAL
+
+        Args:
+            task (Task): an instance of Task
+            task_assignment_timeout (int): timeout value on a client requesting its task
+            task_result_timeout (int): timeout value on reply of one client
+            dynamic_targets (bool): allow clients to join after this task starts
+        """
+        TaskManager.__init__(self)
+        if task_assignment_timeout is None:
+            task_assignment_timeout = 0
+
+        if task_result_timeout is None:
+            task_result_timeout = 0
+
+        task.props[_KEY_DYNAMIC_TARGETS] = dynamic_targets
+        task.props[_KEY_TASK_ASSIGN_TIMEOUT] = task_assignment_timeout
+        task.props[_KEY_TASK_RESULT_TIMEOUT] = task_result_timeout
+        task.props[_KEY_LAST_SEND_IDX] = -1  # client index of last send
+        task.props[_PENDING_CLIENT_TASK] = None
+
+    def check_task_send(self, client_task: ClientTask, fl_ctx: FLContext) -> TaskCheckStatus:
+        """Determine whether the task should be sent to the client.
+
+        Args:
+            client_task (ClientTask): the task processing state of the client
+            fl_ctx (FLContext): fl context that comes with the task request
+
+        Returns:
+            TaskCheckStatus: NO_BLOCK for not sending the task, BLOCK for waiting, SEND for OK to send
+        """
+        client_name = client_task.client.name
+        task = client_task.task
+        if task.props[_KEY_DYNAMIC_TARGETS]:
+            if task.targets is None:
+                task.targets = []
+            if client_name not in task.targets:
+                self.logger.debug(f"{client_name=} added to task.targets")
+                task.targets.append(client_name)
+
+        # is this client eligible?
+        if client_name not in task.targets:
+            # this client is not a target
+            return TaskCheckStatus.NO_BLOCK
+
+        # adjust client window
+        win_start_idx, win_end_idx = self._determine_window(task)
+        self.logger.debug(f"{win_start_idx=} {win_end_idx=}")
+        if win_start_idx < 0:
+            # wait for this task to end by the monitor
+            return TaskCheckStatus.BLOCK
+
+        # see whether this client is in the window
+        for i in range(win_start_idx, win_end_idx):
+            if client_name == task.targets[i]:
+                # this client is in the window!
+                self.logger.debug(f"last_send_idx={i}")
+                task.props[_KEY_LAST_SEND_IDX] = i
+                return TaskCheckStatus.SEND
+
+        # this client is not in the window
+        return TaskCheckStatus.NO_BLOCK
+
+    def _determine_window(self, task: Task) -> Tuple[int, int]:
+        """Return two indexes (starting/ending) of a window of client candidates.
+
+        When starting is negative and ending is 0, the window is closed and the task should exit
+        When both starting and ending are negative, there is no client candidate as current client task has not returned
+        """
+        # adjust client window
+        last_send_idx = task.props[_KEY_LAST_SEND_IDX]
+        task_result_timeout = task.props[_KEY_TASK_RESULT_TIMEOUT]
+        if last_send_idx >= 0:
+            # see whether the result has been received
+            last_task = task.last_client_task_map[task.targets[last_send_idx]]
+            self.logger.debug(f"{last_task=}")
+
+            if last_task.result_received_time is None:
+                # result has not been received
+                # should this client timeout?
+                if task_result_timeout and time.time() - last_task.task_sent_time > task_result_timeout:
+                    # timeout!
+                    # we give up on this client and move to the next target
+                    win_start_idx = last_send_idx + 1
+                    win_start_time = last_task.task_sent_time + task_result_timeout
+                    self.logger.debug(f"client task result timed out {win_start_idx=} {win_start_time=}")
+                else:
+                    # continue to wait
+                    self.logger.debug(f"keep waiting on {task=}")
+                    return -1, -1
+            else:
+                # result has been received!
+                win_start_idx = last_send_idx + 1
+                win_start_time = last_task.result_received_time
+                self.logger.debug(f"result received {win_start_idx=} {win_start_time=}")
+        else:
+            # nothing has been sent
+            win_start_idx = 0
+            win_start_time = task.schedule_time
+            self.logger.debug(f"nothing has been sent {win_start_idx=} {win_start_time=}")
+
+        num_targets = 0 if task.targets is None else len(task.targets)
+        if num_targets and win_start_idx >= num_targets:
+            # we reached the end of targets
+            # so task should exit
+            return -1, 0
+
+        task_assignment_timeout = task.props[_KEY_TASK_ASSIGN_TIMEOUT]
+        if task_assignment_timeout:
+            win_size = int((time.time() - win_start_time) / task_assignment_timeout) + 1
+        else:
+            win_size = 1
+
+        self.logger.debug(f"{win_size=}")
+        win_end_idx = win_start_idx + win_size
+
+        # Should exit if win extends past the entire target list + 1
+        if task_assignment_timeout and win_end_idx > num_targets + 1:
+            return -1, 0
+        if win_end_idx > num_targets:
+            win_end_idx = num_targets
+
+        self.logger.debug(f"{win_end_idx=}")
+        return win_start_idx, win_end_idx
+
+    def check_task_exit(self, task: Task) -> Tuple[bool, TaskCompletionStatus]:
+        """Determine whether the task should exit.
+        Args:
+            task (Task): an instance of Task
+
+        Tuple[bool, TaskCompletionStatus]:
+            first entry in the tuple means whether to exit the task or not.  If it's True, the task should exit.
+            second entry in the tuple indicates the TaskCompletionStatus.
+        """
+        # are we waiting for any client?
+        win_start_idx, win_end_idx = self._determine_window(task)
+        last_send_idx = task.props[_KEY_LAST_SEND_IDX]
+        last_client_task = None
+        if last_send_idx >= 0:
+            # see whether the result has been received
+            last_client_task = task.last_client_task_map[task.targets[last_send_idx]]
+
+        self.logger.debug(f"check_task_exit {win_start_idx=} {win_end_idx=}")
+        if win_start_idx < 0 and win_end_idx == 0:
+            if last_client_task and last_client_task.result_received_time is not None:
+                return True, TaskCompletionStatus.OK
+            return True, TaskCompletionStatus.TIMEOUT
+        else:
+            return False, TaskCompletionStatus.IGNORED
+
+    def check_task_result(self, result: Shareable, client_task: ClientTask, fl_ctx: FLContext):
+        """Check the result received from the client.
+        see whether the client_task is the last one in the task's list
+        If not, then it is a late response and ReservedHeaderKey.REPLY_IS_LATE is
+        set to True in result's header.
+
+        Args:
+            result (Shareable): an instance of Shareable
+            client_task (ClientTask): the task processing state of the client
+            fl_ctx (FLContext): fl context that comes with the task request
+        """
+        # see whether the client_task is the last one in the task's list
+        # If not, then it is a late response
+        task = client_task.task
+        if client_task != task.client_tasks[-1]:
+            result.set_header(key=ReservedHeaderKey.REPLY_IS_LATE, value=True)
diff --git a/nvflare/apis/impl/task_manager.py b/nvflare/apis/impl/task_manager.py
new file mode 100644
index 0000000000..774867e632
--- /dev/null
+++ b/nvflare/apis/impl/task_manager.py
@@ -0,0 +1,95 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from enum import Enum
+from typing import Tuple
+
+from nvflare.apis.controller_spec import ClientTask, Task, TaskCompletionStatus
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+
+class TaskCheckStatus(Enum):
+
+    SEND = 1  # send the task to the client
+    BLOCK = 2  # do not send the task, and block other tasks
+    NO_BLOCK = 3  # do not send the task, and continue checking
+
+
+class TaskManager(object):
+    def __init__(self):
+        """
+        Programming Conventions:
+        A TaskManager should be implemented as a state-free object.
+        All task processing state info should be stored in the Task's props dict.
+        Name the keys in the props dict with prefix "__" to avoid potential conflict with
+        app-defined props.
+        """
+        self._name = self.__class__.__name__
+        self.logger = logging.getLogger(self._name)
+
+    def check_task_send(self, client_task: ClientTask, fl_ctx: FLContext) -> TaskCheckStatus:
+        """Determine whether the task should be sent to the client.
+
+        Default logic:
+        If the client already did the task, don't send again (BLOCK).
+        If the client is in the task's target list or the task's target
+        list is None (meaning all clients), then send the task (SEND). Otherwise, do not block the
+        task checking (NO_BLOCK), so next task will be checked.
+
+        Args:
+            client_task (ClientTask): the task processing state of the client
+            fl_ctx (FLContext): fl context that comes with the task request
+
+
+        Returns:
+            TaskCheckStatus: NO_BLOCK for not sending the task, BLOCK for waiting, SEND for OK to send
+        """
+
+        if client_task.result_received_time:
+            # the task was already sent to the client AND result was already received
+            # do not send again
+            return TaskCheckStatus.NO_BLOCK
+
+        client_name = client_task.client.name
+        if client_task.task.targets is None or client_name in client_task.task.targets:
+            return TaskCheckStatus.SEND
+        else:
+            return TaskCheckStatus.NO_BLOCK
+
+    def check_task_exit(self, task: Task) -> Tuple[bool, TaskCompletionStatus]:
+        """Determine whether the task should exit.
+
+        Args:
+            task (Task): an instance of Task
+
+        Tuple[bool, TaskCompletionStatus]:
+            first entry in the tuple means whether to exit the task or not.  If it's True, the task should exit.
+            second entry in the tuple indicates the TaskCompletionStatus.
+        """
+        pass
+
+    def check_task_result(self, result: Shareable, client_task: ClientTask, fl_ctx: FLContext):
+        """Check the result received from the client.
+
+        The manager can set appropriate headers into the result to indicate certain conditions (e.g.
+        late response).
+
+        Args:
+            result (Shareable): the result to be checked
+            client_task (ClientTask): the task processing state of the client
+            fl_ctx (FLContext): fl context that comes with the task request
+        """
+        pass
diff --git a/nvflare/apis/responder.py b/nvflare/apis/responder.py
new file mode 100644
index 0000000000..77987424b7
--- /dev/null
+++ b/nvflare/apis/responder.py
@@ -0,0 +1,105 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import Tuple
+
+from nvflare.apis.signal import Signal
+
+from .client import Client
+from .fl_component import FLComponent
+from .fl_context import FLContext
+from .shareable import Shareable
+
+
+class Responder(FLComponent, ABC):
+    def __init__(self):
+        FLComponent.__init__(self)
+
+    @abstractmethod
+    def process_task_request(self, client: Client, fl_ctx: FLContext) -> Tuple[str, str, Shareable]:
+        """
+        Called by the Engine when a task request is received from a client.
+
+        Args:
+            client:
+            fl_ctx:
+
+        Returns: task name, task id, and task data
+
+        """
+        pass
+
+    @abstractmethod
+    def handle_exception(self, task_id: str, fl_ctx: FLContext):
+        """
+        Called after process_task_request returns, but exception occurs before task is sent out
+        """
+        pass
+
+    @abstractmethod
+    def process_submission(self, client: Client, task_name: str, task_id: str, result: Shareable, fl_ctx: FLContext):
+        """
+        Called by the Engine to process the submitted result from a client.
+
+        Args:
+            client:
+            task_name:
+            task_id:
+            result:
+            fl_ctx:
+
+        Returns:
+
+        """
+        pass
+
+    def initialize_run(self, fl_ctx: FLContext):
+        """
+        Called when a new RUN is about to start.
+
+        Args:
+            fl_ctx: FL context. It must contain 'run_number' that is to be initialized
+
+        Returns:
+
+        """
+        pass
+
+    @abstractmethod
+    def control_flow(self, abort_signal: Signal, fl_ctx: FLContext):
+        """
+        This is the control logic for the RUN.
+        NOTE: this is running in a separate thread, and its life is the duration of the RUN.
+
+        Args:
+            fl_ctx:
+            abort_signal:
+
+        Returns:
+
+        """
+        pass
+
+    def finalize_run(self, fl_ctx: FLContext):
+        """
+        Called when a new RUN is finished.
+
+        Args:
+            fl_ctx:
+
+        Returns:
+
+        """
+        pass
diff --git a/nvflare/apis/server_engine_spec.py b/nvflare/apis/server_engine_spec.py
new file mode 100644
index 0000000000..aabed6d14a
--- /dev/null
+++ b/nvflare/apis/server_engine_spec.py
@@ -0,0 +1,114 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import List, Tuple
+
+from nvflare.apis.shareable import Shareable
+from nvflare.widgets.widget import Widget
+
+from .client import Client
+from .fl_context import FLContext
+from .workspace import Workspace
+
+
+class ServerEngineSpec(ABC):
+    def fire_event(self, event_type: str, fl_ctx: FLContext):
+        pass
+
+    @abstractmethod
+    def get_clients(self) -> List[Client]:
+        pass
+
+    @abstractmethod
+    def validate_clients(self, client_names: List[str]) -> Tuple[List[Client], List[str]]:
+        """
+        Validate specified client names.
+
+        Args:
+            client_names: list of names to be validated
+
+        Returns: a list of validate clients  and a list of invalid client names
+
+        """
+        pass
+
+    @abstractmethod
+    def new_context(self) -> FLContext:
+        # the engine must use FLContextManager to create a new context!
+        pass
+
+    @abstractmethod
+    def get_workspace(self) -> Workspace:
+        pass
+
+    @abstractmethod
+    def get_component(self, component_id: str) -> object:
+        pass
+
+    @abstractmethod
+    def register_aux_message_handler(self, topic: str, message_handle_func):
+        """
+        Register aux message handling function with specified topics.
+
+        Args:
+            topic: the topic to be handled by the func
+            message_handle_func: the func to handle the message. Must follow aux_message_handle_func_signature.
+
+        Returns:
+
+        Exception is raised when:
+            a handler is already registered for the topic;
+            bad topic - must be a non-empty string
+            bad message_handle_func - must be callable
+
+        Implementation Note:
+        This method should simply call the ServerAuxRunner's register_aux_message_handler method.
+        """
+        pass
+
+    @abstractmethod
+    def send_aux_request(self, targets: [], topic: str, request: Shareable, timeout: float, fl_ctx: FLContext) -> dict:
+        """
+        Send a request to specified clients via the aux channel.
+
+        Args:
+            targets: target clients. None or empty list means all clients
+            topic: topic of the request
+            request: request to be sent
+            timeout: number of secs to wait for replies. 0 means fire-and-forget.
+            fl_ctx: FL context
+
+        Returns: a dict of replies (client name => reply Shareable)
+
+        Implementation: simply calls the ServerAuxRunner's send_aux_request method.
+
+        """
+        pass
+
+    def fire_and_forget_aux_request(self, targets: [], topic: str, request: Shareable, fl_ctx: FLContext) -> dict:
+        return self.send_aux_request(targets, topic, request, 0.0, fl_ctx)
+
+    @abstractmethod
+    def get_widget(self, widget_id: str) -> Widget:
+        """
+        Get the widget with the specified ID
+
+        Args:
+            widget_id: ID of the widget
+
+        Returns: the widget or None if not found
+
+        """
+        pass
diff --git a/nvflare/apis/shareable.py b/nvflare/apis/shareable.py
new file mode 100644
index 0000000000..0d681d6f27
--- /dev/null
+++ b/nvflare/apis/shareable.py
@@ -0,0 +1,137 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pickle
+
+from .fl_constant import ReservedKey, ReturnCode
+
+
+class ReservedHeaderKey(object):
+
+    HEADERS = "__headers__"
+    TOPIC = "__topic__"
+    RC = ReservedKey.RC
+    COOKIE_JAR = ReservedKey.COOKIE_JAR
+    PEER_PROPS = "__peer_props__"
+    REPLY_IS_LATE = "__reply_is_late__"
+    TASK_NAME = ReservedKey.TASK_NAME
+    TASK_ID = ReservedKey.TASK_ID
+    WORKFLOW = ReservedKey.WORKFLOW
+    CONTENT_TYPE = "__content_type__"
+
+
+class Shareable(dict):
+    """The information communicated between server and client
+
+    Shareable is just a dict that can have any keys and values, defined by developers and users.
+    It is recommended that keys are strings. Values must be serializable.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self[ReservedHeaderKey.HEADERS] = {}
+
+    def set_header(self, key: str, value):
+        header = self.get(ReservedHeaderKey.HEADERS, None)
+        if not header:
+            header = {}
+            self[ReservedHeaderKey.HEADERS] = header
+        header[key] = value
+
+    def get_header(self, key: str, default=None):
+        header = self.get(ReservedHeaderKey.HEADERS, None)
+        if not header:
+            return default
+        else:
+            if not isinstance(header, dict):
+                raise ValueError("header object must be a dict, but got {}".format(type(header)))
+            return header.get(key, default)
+
+    # some convenience methods
+    def get_return_code(self, default=ReturnCode.OK):
+        return self.get_header(ReservedHeaderKey.RC, default)
+
+    def set_return_code(self, rc):
+        self.set_header(ReservedHeaderKey.RC, rc)
+
+    def add_cookie(self, name: str, data):
+        """
+        Add a cookie that is to be sent to the client and echoed back in response.
+        This method is intended to be called by the Server side.
+
+        Args:
+            name: the name of the cookie
+            data: the data of the cookie, which must be serializable
+
+        Returns:
+
+        """
+        cookie_jar = self.get_cookie_jar()
+        if not cookie_jar:
+            cookie_jar = {}
+            self.set_header(key=ReservedHeaderKey.COOKIE_JAR, value=cookie_jar)
+        cookie_jar[name] = data
+
+    def get_cookie_jar(self):
+        return self.get_header(key=ReservedHeaderKey.COOKIE_JAR, default=None)
+
+    def set_cookie_jar(self, jar):
+        self.set_header(key=ReservedHeaderKey.COOKIE_JAR, value=jar)
+
+    def get_cookie(self, name: str, default=None):
+        jar = self.get_cookie_jar()
+        if not jar:
+            return default
+        return jar.get(name, default)
+
+    def set_peer_props(self, props: dict):
+        self.set_header(ReservedHeaderKey.PEER_PROPS, props)
+
+    def get_peer_props(self):
+        return self.get_header(ReservedHeaderKey.PEER_PROPS, None)
+
+    def get_peer_prop(self, key: str, default):
+        props = self.get_peer_props()
+        if not isinstance(props, dict):
+            return default
+        return props.get(key, default)
+
+    def to_bytes(self) -> bytes:
+        """method to serialize the Model object into bytes.
+
+        Returns:
+            object serialized in bytes.
+
+        """
+        return pickle.dumps(self)
+
+    @classmethod
+    def from_bytes(cls, data: bytes):
+        """method to convert the object bytes into Model object.
+
+        Args:
+            data: a bytes object
+
+        Returns:
+            an object loaded by pickle from data
+
+        """
+        return pickle.loads(data)
+
+
+# some convenience functions
+def make_reply(rc) -> Shareable:
+    reply = Shareable()
+    reply.set_return_code(rc)
+    return reply
diff --git a/nvflare/apis/signal.py b/nvflare/apis/signal.py
new file mode 100644
index 0000000000..6c3d29b15a
--- /dev/null
+++ b/nvflare/apis/signal.py
@@ -0,0 +1,32 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+
+class Signal(object):
+    def __init__(self):
+        self.value = None
+        self.trigger_time = None
+        self.triggered = False
+
+    def trigger(self, value):
+        self.value = value
+        self.trigger_time = time.time()
+        self.triggered = True
+
+    def reset(self, value=None):
+        self.value = value
+        self.trigger_time = None
+        self.triggered = False
diff --git a/nvflare/apis/utils/__init__.py b/nvflare/apis/utils/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/apis/utils/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/apis/utils/fl_context_utils.py b/nvflare/apis/utils/fl_context_utils.py
new file mode 100644
index 0000000000..baf867e6b4
--- /dev/null
+++ b/nvflare/apis/utils/fl_context_utils.py
@@ -0,0 +1,74 @@
+import logging
+import pickle
+
+from nvflare.apis.fl_constant import FLContextKey, NonSerializableKeys
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+
+def get_serializable_data(fl_ctx: FLContext):
+    logger = logging.getLogger("fl_context_utils")
+    new_fl_ctx = FLContext()
+    for k, v in fl_ctx.props.items():
+        if k not in NonSerializableKeys.KEYS:
+            try:
+                pickle.dumps(v)
+                new_fl_ctx.props[k] = v
+            except:
+                logger.warning(generate_log_message(fl_ctx, f"Object is not serializable (discarded): {k} - {v}"))
+    return new_fl_ctx
+
+
+def generate_log_message(fl_ctx: FLContext, msg: str):
+    _my_run = "run"
+    _peer_run = "peer_run"
+    _peer_name = "peer"
+    _task_name = "task_name"
+    _task_id = "task_id"
+    _rc = "peer_rc"
+    _wf = "wf"
+
+    all_kvs = {}
+    my_run = fl_ctx.get_run_number()
+    if not my_run:
+        my_run = "?"
+    all_kvs[_my_run] = my_run
+
+    task_name = fl_ctx.get_prop(FLContextKey.TASK_NAME, None)
+    task_id = fl_ctx.get_prop(FLContextKey.TASK_ID, None)
+
+    if task_name:
+        all_kvs[_task_name] = task_name
+
+    if task_id:
+        all_kvs[_task_id] = task_id
+
+    wf_id = fl_ctx.get_prop(FLContextKey.WORKFLOW, None)
+    if wf_id is not None:
+        all_kvs[_wf] = wf_id
+
+    peer_ctx = fl_ctx.get_peer_context()
+    if peer_ctx:
+        assert isinstance(peer_ctx, FLContext)
+        peer_run = peer_ctx.get_run_number()
+        if not peer_run:
+            peer_run = "?"
+        all_kvs[_peer_run] = peer_run
+
+        peer_name = peer_ctx.get_identity_name()
+        if not peer_name:
+            peer_name = "?"
+        all_kvs[_peer_name] = peer_name
+
+    reply = fl_ctx.get_prop(FLContextKey.REPLY, None)
+    if isinstance(reply, Shareable):
+        rc = reply.get_return_code("OK")
+        all_kvs[_rc] = rc
+
+    item_order = [_my_run, _wf, _peer_name, _peer_run, _rc, _task_name, _task_id]
+    ctx_items = []
+    for item in item_order:
+        if item in all_kvs:
+            ctx_items.append(item + "=" + str(all_kvs[item]))
+
+    return "[" + ", ".join(ctx_items) + "]: " + msg
diff --git a/nvflare/apis/utils/format_check.py b/nvflare/apis/utils/format_check.py
new file mode 100644
index 0000000000..aa26f35897
--- /dev/null
+++ b/nvflare/apis/utils/format_check.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+type_pattern_mapping = {
+    "server": r"^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$",
+    "client": r"^[A-Za-z0-9-_]+$",
+    "admin": r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}$",
+    "email": r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}$",
+    "org": r"^[A-Za-z0-9_]+$",
+}
+
+
+def name_check(name: str, entity_type: str):
+    regex_pattern = type_pattern_mapping.get(entity_type)
+    if regex_pattern is None:
+        return True, f"{entity_type=} not defined, unable to check {name=}."
+    if re.match(regex_pattern, name):
+        return False, f"{name=} passed on {regex_pattern=} check"
+    else:
+        return True, f"{name=} is ill-formatted based on {regex_pattern=}"
diff --git a/nvflare/apis/workspace.py b/nvflare/apis/workspace.py
new file mode 100644
index 0000000000..7daaf1355d
--- /dev/null
+++ b/nvflare/apis/workspace.py
@@ -0,0 +1,58 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Workspace folder structure:
+
+WSROOT
+    startup (optional)
+        provisioned content
+    runs
+        1
+            config (required)
+                config_fed_client.json
+                config_fed_server.json
+                ...
+            custom (optional)
+                custom python code
+            whatever (app defined)
+
+"""
+
+import os
+
+
+class Workspace(object):
+    def __init__(self, root_dir: str, name: str, config_folder: str):
+        self.root_dir = root_dir
+        self.name = name
+        self.config_folder = config_folder
+
+    def get_startup_kit_dir(self) -> str:
+        return os.path.join(self.root_dir, "startup")
+
+    def get_root_dir(self) -> str:
+        return self.root_dir
+
+    def get_run_dir(self, run_num: int) -> str:
+        return os.path.join(self.root_dir, "run_" + str(run_num))
+
+    def get_app_dir(self, run_num: int) -> str:
+        return os.path.join(self.get_run_dir(run_num), "app_" + self.name)
+
+    def get_app_config_dir(self, run_num) -> str:
+        return os.path.join(self.get_app_dir(run_num), self.config_folder)
+
+    def get_app_custom_dir(self, run_num) -> str:
+        return os.path.join(self.get_app_dir(run_num), "custom")
diff --git a/nvflare/app_common/__init__.py b/nvflare/app_common/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/app_common/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/app_common/abstract/__init__.py b/nvflare/app_common/abstract/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/app_common/abstract/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/app_common/abstract/aggregator.py b/nvflare/app_common/abstract/aggregator.py
new file mode 100644
index 0000000000..1bea7b14d8
--- /dev/null
+++ b/nvflare/app_common/abstract/aggregator.py
@@ -0,0 +1,49 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+
+class Aggregator(FLComponent, ABC):
+    @abstractmethod
+    def accept(self, shareable: Shareable, fl_ctx: FLContext) -> bool:
+        """
+            accept the shareable submitted by the client.
+
+        Args:
+            shareable: submitted Shareable object
+            fl_ctx: FLContext
+
+        Returns:
+            first boolean to indicate if the contribution has been accepted.
+
+        """
+        pass
+
+    @abstractmethod
+    def aggregate(self, fl_ctx: FLContext) -> Shareable:
+        """
+            perform the aggregation for all the received Shareable from the clients.
+
+        Args:
+            fl_ctx: FLContext
+
+        Returns:
+            shareable
+        """
+        pass
diff --git a/nvflare/app_common/abstract/formatter.py b/nvflare/app_common/abstract/formatter.py
new file mode 100644
index 0000000000..60ead341e9
--- /dev/null
+++ b/nvflare/app_common/abstract/formatter.py
@@ -0,0 +1,35 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import abstractmethod
+
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_context import FLContext
+
+
+class Formatter(FLComponent):
+    def __init__(self) -> None:
+        super(Formatter, self).__init__()
+
+    @abstractmethod
+    def format(self, fl_ctx: FLContext) -> str:
+        """Format the data into human readable string for
+
+        Args:
+            fl_ctx (FLContext): FL Context object.
+
+        Returns:
+            str: Human readable string.
+        """
+        pass
diff --git a/nvflare/app_common/abstract/learnable.py b/nvflare/app_common/abstract/learnable.py
new file mode 100644
index 0000000000..6f22fe2bfd
--- /dev/null
+++ b/nvflare/app_common/abstract/learnable.py
@@ -0,0 +1,46 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# from __future__ import annotations
+
+import pickle
+
+
+class Learnable(dict):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def to_bytes(self) -> bytes:
+        """
+            method to serialize the Learnable object into bytes.
+
+        Returns:
+            object serialized in bytes.
+
+        """
+        return pickle.dumps(self)
+
+    @classmethod
+    def from_bytes(cls, data: bytes):
+        """
+            method to convert the object bytes into Learnable object.
+
+        Args:
+            data: a bytes object
+
+        Returns:
+            an object loaded by pickle from data
+
+        """
+        return pickle.loads(data)
diff --git a/nvflare/app_common/abstract/learnable_persistor.py b/nvflare/app_common/abstract/learnable_persistor.py
new file mode 100644
index 0000000000..c8cbb68e72
--- /dev/null
+++ b/nvflare/app_common/abstract/learnable_persistor.py
@@ -0,0 +1,47 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.learnable import Learnable
+
+
+class LearnablePersistor(FLComponent, ABC):
+    @abstractmethod
+    def load(self, fl_ctx: FLContext) -> Learnable:
+        """
+            load the Learnable object.
+
+        Args:
+            fl_ctx: FLContext
+
+        Returns:
+            Learnable object loaded
+
+        """
+        pass
+
+    @abstractmethod
+    def save(self, learnable: Learnable, fl_ctx: FLContext):
+        """
+            persist the Learnable object
+
+        Args:
+            learnable: the Learnable object to be saved
+            fl_ctx: FLContext
+
+        """
+        pass
diff --git a/nvflare/app_common/abstract/model.py b/nvflare/app_common/abstract/model.py
new file mode 100644
index 0000000000..6a08d0d91c
--- /dev/null
+++ b/nvflare/app_common/abstract/model.py
@@ -0,0 +1,70 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+
+The Learnable in the deep learning domain is usually called Model by researchers.
+This import simply lets you call the Learnable 'Model'.
+
+"""
+from nvflare.apis.dxo import DXO, DataKind
+
+from .learnable import Learnable as ModelLearnable
+
+
+class ModelLearnableKey(object):
+
+    """
+    Model Learnable is a dict that contains two items: weights and meta info
+    """
+
+    WEIGHTS = "weights"
+    META = "meta"
+
+
+def validate_model_learnable(model_learnable: ModelLearnable) -> str:
+    """
+    Check whether the specified model is a valid Model Shareable
+
+    Args:
+        model: model to be validated
+
+    Returns: error text
+
+    """
+    if not isinstance(model_learnable, ModelLearnable):
+        return "invalid model learnable: expect Model type but got {}".format(type(model_learnable))
+
+    if ModelLearnableKey.WEIGHTS not in model_learnable:
+        return "invalid model learnable: missing weights"
+
+    if ModelLearnableKey.META not in model_learnable:
+        return "invalid model learnable: missing meta"
+
+    return ""
+
+
+def make_model_learnable(weights, meta_props) -> ModelLearnable:
+    ml = ModelLearnable()
+    ml[ModelLearnableKey.WEIGHTS] = weights
+    ml[ModelLearnableKey.META] = meta_props
+    return ml
+
+
+def model_learnable_to_dxo(ml: ModelLearnable) -> DXO:
+    err = validate_model_learnable(ml)
+    if err:
+        raise ValueError(err)
+
+    return DXO(data_kind=DataKind.WEIGHTS, data=ml[ModelLearnableKey.WEIGHTS], meta=ml[ModelLearnableKey.META])
diff --git a/nvflare/app_common/abstract/model_locator.py b/nvflare/app_common/abstract/model_locator.py
new file mode 100644
index 0000000000..3e6142d318
--- /dev/null
+++ b/nvflare/app_common/abstract/model_locator.py
@@ -0,0 +1,47 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from nvflare.apis.dxo import DXO
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_context import FLContext
+
+
+class ModelLocator(FLComponent):
+    def __init__(self):
+        super(ModelLocator, self).__init__()
+
+    def get_model_names(self, fl_ctx: FLContext) -> List[str]:
+        """List the name of the models
+
+        Args:
+            fl_ctx (FLContext): FL Context object
+
+        Returns:
+            List[str]: List of names for models
+        """
+        pass
+
+    def locate_model(self, model_name, fl_ctx: FLContext) -> DXO:
+        """Locate a single model by it's name
+
+        Args:
+            model_name (str): Name of the model.
+            fl_ctx (FLContext): FL Context object.
+
+        Returns:
+            DXO: a DXO object
+        """
+        pass
diff --git a/nvflare/app_common/abstract/model_persistor.py b/nvflare/app_common/abstract/model_persistor.py
new file mode 100644
index 0000000000..e99e305f9c
--- /dev/null
+++ b/nvflare/app_common/abstract/model_persistor.py
@@ -0,0 +1,54 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable
+
+from .learnable_persistor import LearnablePersistor
+
+
+class ModelPersistor(LearnablePersistor, ABC):
+    def load(self, fl_ctx: FLContext):
+        return self.load_model(fl_ctx)
+
+    def save(self, learnable: ModelLearnable, fl_ctx: FLContext):
+        self.save_model(learnable, fl_ctx)
+
+    @abstractmethod
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        """
+            initialize and load the model.
+
+        Args:
+            fl_ctx: FLContext
+
+        Returns:
+            Model object
+
+        """
+        pass
+
+    @abstractmethod
+    def save_model(self, model: ModelLearnable, fl_ctx: FLContext):
+        """
+            persist the model object
+
+        Args:
+            model: Model object to be saved
+            fl_ctx: FLContext
+
+        """
+        pass
diff --git a/nvflare/app_common/abstract/model_processor.py b/nvflare/app_common/abstract/model_processor.py
new file mode 100644
index 0000000000..be120a7ad7
--- /dev/null
+++ b/nvflare/app_common/abstract/model_processor.py
@@ -0,0 +1,35 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+from nvflare.apis.fl_context import FLContext
+
+
+class ModelProcessor(ABC):
+    @abstractmethod
+    def initialize(self, trainer):
+        pass
+
+    @abstractmethod
+    def extract_model(self, model_vars, fl_ctx: FLContext):
+        pass
+
+    @abstractmethod
+    def apply_model(self, model_params, fl_ctx: FLContext, options=None):
+        pass
+
+    @abstractmethod
+    def get_local_models(self, model_params=None):
+        pass
diff --git a/nvflare/app_common/abstract/shareable_generator.py b/nvflare/app_common/abstract/shareable_generator.py
new file mode 100644
index 0000000000..582ba1e466
--- /dev/null
+++ b/nvflare/app_common/abstract/shareable_generator.py
@@ -0,0 +1,52 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.abstract.learnable import Learnable
+
+
+class ShareableGenerator(FLComponent, ABC):
+    @abstractmethod
+    def learnable_to_shareable(self, model: Learnable, fl_ctx: FLContext) -> Shareable:
+        """
+            generate the initial Shareable from the Learnable object.
+
+        Args:
+            model: model object
+            fl_ctx: FLContext
+
+        Returns:
+            shareable
+
+        """
+        pass
+
+    @abstractmethod
+    def shareable_to_learnable(self, shareable: Shareable, fl_ctx: FLContext) -> Learnable:
+        """
+            construct the Learnable object from Shareable
+
+        Args:
+            shareable: shareable
+            fl_ctx: FLContext
+
+        Returns:
+            model object
+
+        """
+        pass
diff --git a/nvflare/app_common/aggregators/__init__.py b/nvflare/app_common/aggregators/__init__.py
new file mode 100644
index 0000000000..11da76d441
--- /dev/null
+++ b/nvflare/app_common/aggregators/__init__.py
@@ -0,0 +1,18 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .accumulate_model_aggregator import AccumulateWeightedAggregator
+from .intime_accumulate_model_aggregator import InTimeAccumulateWeightedAggregator
+
+__all__ = ["AccumulateWeightedAggregator", "InTimeAccumulateWeightedAggregator"]
diff --git a/nvflare/app_common/aggregators/accumulate_model_aggregator.py b/nvflare/app_common/aggregators/accumulate_model_aggregator.py
new file mode 100644
index 0000000000..d08722fcfd
--- /dev/null
+++ b/nvflare/app_common/aggregators/accumulate_model_aggregator.py
@@ -0,0 +1,189 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import numpy as np
+
+from nvflare.apis.dxo import DXO, DataKind, MetaKey, from_shareable
+from nvflare.apis.fl_constant import ReservedKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.abstract.aggregator import Aggregator
+from nvflare.app_common.app_constant import AppConstants
+
+
+class AccuItem(object):
+    def __init__(self, client, data, steps):
+        self.client = client
+        self.data = data
+        self.steps = steps
+
+
+class AccumulateWeightedAggregator(Aggregator):
+    def __init__(self, exclude_vars=None, aggregation_weights=None, expected_data_kind="WEIGHT_DIFF"):
+        super().__init__()
+        self.exclude_vars = re.compile(exclude_vars) if exclude_vars else None
+        self.aggregation_weights = aggregation_weights or {}
+        if expected_data_kind not in [DataKind.WEIGHT_DIFF, DataKind.WEIGHTS]:
+            raise ValueError(f"{expected_data_kind=} not in WEIGHT_DIFF or WEIGHTS")
+        self.expected_data_kind = expected_data_kind
+        self.logger.debug(f"aggregation weights control: {aggregation_weights}")
+
+        self.accumulator = []
+
+        self.warning_count = dict()
+        self.warning_limit = 10
+
+    def accept(self, shareable: Shareable, fl_ctx: FLContext) -> bool:
+        try:
+            dxo = from_shareable(shareable)
+        except:
+            self.log_exception(fl_ctx, "shareable data is not a valid DXO")
+            return False
+        assert isinstance(dxo, DXO)
+
+        if dxo.data_kind not in (DataKind.WEIGHT_DIFF, DataKind.WEIGHTS):
+            self.log_error(fl_ctx, "cannot handle data kind {}".format(dxo.data_kind))
+            return False
+
+        if dxo.data_kind != self.expected_data_kind:
+            self.log_error(fl_ctx, "expect {} but got {}".format(self.expected_data_kind, dxo.data_kind))
+            return False
+
+        if (processed_algorithm := dxo.get_meta_prop(MetaKey.PROCESSED_ALGORITHM)) is not None:
+            self.log_error(fl_ctx, f"unable to accept shareable processed by {processed_algorithm}")
+            return False
+
+        current_round = fl_ctx.get_prop(AppConstants.CURRENT_ROUND)
+        self.log_debug(fl_ctx, f"current_round: {current_round}")
+        client_name = shareable.get_peer_prop(key=ReservedKey.IDENTITY_NAME, default="?")
+        contribution_round = shareable.get_header(AppConstants.CONTRIBUTION_ROUND)
+
+        rc = shareable.get_return_code()
+        if rc and rc != ReturnCode.OK:
+            self.log_info(fl_ctx, f"Client {client_name} returned rc: {rc}. Disregarding contribution.")
+            return False
+
+        data = dxo.data
+        if data is None:
+            self.log_error(fl_ctx, "no data to aggregate")
+            return False
+
+        n_iter = dxo.get_meta_prop(MetaKey.NUM_STEPS_CURRENT_ROUND)
+
+        if contribution_round == current_round:
+            if not self._client_in_accumulator(client_name):
+                self.accumulator.append(AccuItem(client_name, data, n_iter))
+                accepted = True
+            else:
+                self.log_info(
+                    fl_ctx,
+                    f"Discarded: Current round: {current_round} contributions already include client: {client_name}",
+                )
+                accepted = False
+        else:
+            self.log_info(
+                fl_ctx,
+                "Discarded the contribution from {} for round: {}. Current round is: {}".format(
+                    client_name, contribution_round, current_round
+                ),
+            )
+            accepted = False
+        return accepted
+
+    def _client_in_accumulator(self, client_name):
+        return any(client_name == item.client for item in self.accumulator)
+
+    def aggregate(self, fl_ctx: FLContext) -> Shareable:
+        """
+        Aggregate model variables.
+        This function is not thread-safe.
+
+        :return Return True to indicates the current model is the best model so far.
+        """
+        current_round = fl_ctx.get_prop(AppConstants.CURRENT_ROUND)
+        self.log_info(fl_ctx, "aggregating {} updates at round {}".format(len(self.accumulator), current_round))
+
+        # TODO: What if AppConstants.GLOBAL_MODEL is None?
+        acc_vars = [set(acc.data.keys()) for acc in self.accumulator]
+        acc_vars = set.union(*acc_vars) if acc_vars else acc_vars
+        # update vars that are not in exclude pattern
+        vars_to_aggregate = (
+            [g_var for g_var in acc_vars if not self.exclude_vars.search(g_var)] if self.exclude_vars else acc_vars
+        )
+
+        clients_with_messages = []
+        aggregated_model = {}
+        for v_name in vars_to_aggregate:
+            n_local_iters, np_vars = [], []
+            for item in self.accumulator:
+                assert isinstance(item, AccuItem)
+                client_name = item.client
+                data = item.data
+                n_iter = item.steps
+
+                if n_iter is None:
+                    if self.warning_count.get(client_name, 0) <= self.warning_limit:
+                        self.log_warning(
+                            fl_ctx,
+                            f"NUM_STEPS_CURRENT_ROUND missing in meta of shareable"
+                            f" from {client_name} and set to default value, 1.0. "
+                            f" This kind of message will show {self.warning_limit} times at most.",
+                        )
+                        if client_name in self.warning_count:
+                            self.warning_count[client_name] = self.warning_count[client_name] + 1
+                        else:
+                            self.warning_count[client_name] = 0
+                    n_iter = 1.0
+                if v_name not in data.keys():
+                    continue  # this acc doesn't have the variable from client
+                float_n_iter = float(n_iter)
+                n_local_iters.append(float_n_iter)
+                aggregation_weight = self.aggregation_weights.get(client_name)
+                if aggregation_weight is None:
+                    if self.warning_count.get(client_name, 0) <= self.warning_limit:
+                        self.log_warning(
+                            fl_ctx,
+                            f"Aggregation_weight missing for {client_name} and set to default value, 1.0"
+                            f" This kind of message will show {self.warning_limit} times at most.",
+                        )
+                        if client_name in self.warning_count:
+                            self.warning_count[client_name] = self.warning_count[client_name] + 1
+                        else:
+                            self.warning_count[client_name] = 0
+                    aggregation_weight = 1.0
+
+                weighted_value = data[v_name] * float_n_iter * aggregation_weight
+                if client_name not in clients_with_messages:
+                    if client_name in self.aggregation_weights.keys():
+                        self.log_debug(fl_ctx, f"Client {client_name} use weight {aggregation_weight} for aggregation.")
+                    else:
+                        self.log_debug(
+                            fl_ctx,
+                            f"Client {client_name} not defined in the aggregation weight list. Use default value 1.0",
+                        )
+                    clients_with_messages.append(client_name)
+                np_vars.append(weighted_value)
+            if not n_local_iters:
+                continue  # all acc didn't receive the variable from clients
+            new_val = np.sum(np_vars, axis=0) / np.sum(n_local_iters)
+            aggregated_model[v_name] = new_val
+
+        self.accumulator.clear()
+
+        self.log_debug(fl_ctx, f"Model after aggregation: {aggregated_model}")
+
+        dxo = DXO(data_kind=self.expected_data_kind, data=aggregated_model)
+        return dxo.to_shareable()
diff --git a/nvflare/app_common/aggregators/intime_accumulate_model_aggregator.py b/nvflare/app_common/aggregators/intime_accumulate_model_aggregator.py
new file mode 100644
index 0000000000..9c65e9dddc
--- /dev/null
+++ b/nvflare/app_common/aggregators/intime_accumulate_model_aggregator.py
@@ -0,0 +1,190 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from nvflare.apis.dxo import DXO, DataKind, MetaKey, from_shareable
+from nvflare.apis.fl_constant import ReservedKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.abstract.aggregator import Aggregator
+from nvflare.app_common.app_constant import AppConstants
+
+
+class InTimeAccumulateWeightedAggregator(Aggregator):
+    def __init__(self, exclude_vars=None, aggregation_weights=None, expected_data_kind=DataKind.WEIGHT_DIFF):
+        """Perform accumulated weighted aggregation
+        It computes
+        weighted_sum = sum(shareable*n_iteration*aggregation_weights) and
+        sum_of_weights = sum(n_iteration)
+
+        in accept function
+        The aggregate function returns
+        weighted_sum / sum_of_weights
+
+        Args:
+            exclude_vars ([type], optional): regex to match excluded vars during aggregation. Defaults to None.
+            aggregation_weights ([type], optional): dictionary to map client name to its aggregation weights. Defaults to None.
+        """
+        super().__init__()
+        self.exclude_vars = re.compile(exclude_vars) if exclude_vars else None
+        self.aggregation_weights = aggregation_weights or {}
+        if expected_data_kind not in [DataKind.WEIGHT_DIFF, DataKind.WEIGHTS]:
+            raise ValueError(f"{expected_data_kind=} not in WEIGHT_DIFF or WEIGHTS")
+        self.expected_data_kind = expected_data_kind
+        self.logger.debug(f"aggregation weights control: {aggregation_weights}")
+        self.reset_stats()
+        self.warning_count = {}
+        self.warning_limit = 10
+        self.total = dict()
+        self.counts = dict()
+        self.history = list()
+
+    def reset_stats(self):
+        self.total = {}
+        self.counts = {}
+        self.history = []
+
+    def accept(self, shareable: Shareable, fl_ctx: FLContext) -> bool:
+        """Store shareable and update aggregator's internal state
+
+        Args:
+            shareable: information from client
+            fl_ctx: context provided by workflow
+
+        Returns:
+            The first boolean indicates if this shareable is accepted.
+            The second bollean indicates if aggregate can be called.
+        """
+        try:
+            dxo = from_shareable(shareable)
+        except:
+            self.log_exception(fl_ctx, "shareable data is not a valid DXO")
+            return False
+        assert isinstance(dxo, DXO)
+
+        if dxo.data_kind not in (DataKind.WEIGHT_DIFF, DataKind.WEIGHTS):
+            self.log_error(fl_ctx, "cannot handle data kind {}".format(dxo.data_kind))
+            return False
+
+        if dxo.data_kind != self.expected_data_kind:
+            self.log_error(fl_ctx, "expect {} but got {}".format(self.expected_data_kind, dxo.data_kind))
+            return False
+
+        if (processed_algorithm := dxo.get_meta_prop(MetaKey.PROCESSED_ALGORITHM)) is not None:
+            self.log_error(fl_ctx, f"unable to accept shareable processed by {processed_algorithm}")
+            return False
+
+        current_round = fl_ctx.get_prop(AppConstants.CURRENT_ROUND)
+        self.log_debug(fl_ctx, f"current_round: {current_round}")
+        client_name = shareable.get_peer_prop(key=ReservedKey.IDENTITY_NAME, default="?")
+        contribution_round = shareable.get_header(AppConstants.CONTRIBUTION_ROUND)
+
+        rc = shareable.get_return_code()
+        if rc and rc != ReturnCode.OK:
+            self.log_info(fl_ctx, f"Client {client_name} returned rc: {rc}. Disregarding contribution.")
+            return False
+
+        data = dxo.data
+        if data is None:
+            self.log_error(fl_ctx, "no data to aggregate")
+            return False
+
+        n_iter = dxo.get_meta_prop(MetaKey.NUM_STEPS_CURRENT_ROUND)
+        if contribution_round != current_round:
+            self.log_info(
+                fl_ctx,
+                f"discarding shareable from {client_name} at round: {contribution_round}. Current round is: {current_round}",
+            )
+            return False
+
+        for item in self.history:
+            if client_name == item["client_name"]:
+                prev_round = item["round"]
+                self.log_info(
+                    fl_ctx,
+                    f"discarding shareable from {client_name} at round: {contribution_round} as {prev_round} accepted already",
+                )
+                return False
+
+        if n_iter is None:
+            if self.warning_count.get(client_name, 0) <= self.warning_limit:
+                self.log_warning(
+                    fl_ctx,
+                    f"NUM_STEPS_CURRENT_ROUND missing in meta of shareable"
+                    f" from {client_name} and set to default value, 1.0. "
+                    f" This kind of message will show {self.warning_limit} times at most.",
+                )
+                if client_name in self.warning_count:
+                    self.warning_count[client_name] = self.warning_count[client_name] + 1
+                else:
+                    self.warning_count[client_name] = 0
+            n_iter = 1.0
+        float_n_iter = float(n_iter)
+        aggregation_weight = self.aggregation_weights.get(client_name)
+        if aggregation_weight is None:
+            if self.warning_count.get(client_name, 0) <= self.warning_limit:
+                self.log_warning(
+                    fl_ctx,
+                    f"Aggregation_weight missing for {client_name} and set to default value, 1.0"
+                    f" This kind of message will show {self.warning_limit} times at most.",
+                )
+                if client_name in self.warning_count:
+                    self.warning_count[client_name] = self.warning_count[client_name] + 1
+                else:
+                    self.warning_count[client_name] = 0
+            aggregation_weight = 1.0
+
+        for k, v in data.items():
+            if self.exclude_vars is not None and self.exclude_vars.search(k):
+                continue
+            weighted_value = v * aggregation_weight * float_n_iter
+            current_total = self.total.get(k, None)
+            if current_total is None:
+                self.total[k] = weighted_value
+                self.counts[k] = n_iter
+            else:
+                self.total[k] = current_total + weighted_value
+                self.counts[k] = self.counts[k] + n_iter
+        self.history.append(
+            {
+                "client_name": client_name,
+                "round": contribution_round,
+                "aggregation_weight": aggregation_weight,
+                "n_iter": n_iter,
+            }
+        )
+        self.log_debug(fl_ctx, "End accept")
+        return True
+
+    def aggregate(self, fl_ctx: FLContext) -> Shareable:
+        """Called when workflow determines to generate shareable to send back to clients
+
+        Args:
+            fl_ctx (FLContext): context provided by workflow
+
+        Returns:
+            Shareable: the weighted mean of accepted shareables from clients
+        """
+
+        self.log_debug(fl_ctx, "Start aggregation")
+        current_round = fl_ctx.get_prop(AppConstants.CURRENT_ROUND)
+        self.log_info(fl_ctx, f"aggregating {len(self.history)} update(s) at round {current_round}")
+        self.log_debug(fl_ctx, f"complete history {self.history}")
+        aggregated_dict = {k: v / self.counts[k] for k, v in self.total.items()}
+        self.reset_stats()
+        self.log_debug(fl_ctx, "End aggregation")
+
+        dxo = DXO(data_kind=self.expected_data_kind, data=aggregated_dict)
+        return dxo.to_shareable()
diff --git a/nvflare/app_common/app_constant.py b/nvflare/app_common/app_constant.py
new file mode 100644
index 0000000000..652d523d71
--- /dev/null
+++ b/nvflare/app_common/app_constant.py
@@ -0,0 +1,127 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class ExecutorTasks:
+
+    TRAIN = "train"
+    VALIDATE = "validate"
+    CROSS_VALIDATION = "__cross_validation"
+    SUBMIT_BEST = "__submit_best"
+    REPORT_STATUS = "report_status"
+
+
+class AppConstants(object):
+
+    CONFIG_PATH = "config_path"
+    MODEL_NETWORK = "model_network"
+    MULTI_GPU = "multi_gpu"
+    TRAIN_CONTEXT = "train_context"
+    DEVICE = "device"
+    MODEL_NAME = "model_name"
+    MODEL_URL = "model_url"
+    START_ROUND = "start_round"
+    CURRENT_ROUND = "current_round"
+    CONTRIBUTION_ROUND = "contribution_round"
+    CONTRIBUTION_CLIENT = "contribution_client"
+    NUM_ROUNDS = "num_rounds"
+    WAIT_AFTER_MIN_CLIENTS = "wait_after_min_clients"
+
+    NUM_TOTAL_STEPS = "num_total_steps"  # TOTAL_STEPS
+    NUM_EPOCHS_CURRENT_ROUND = "num_epochs_current_round"  # CURRENT_EPOCHS
+    NUM_TOTAL_EPOCHS = "num_total_epochs"  # LOCAL_EPOCHS
+    LOCAL_EPOCHS = "local_epochs"
+
+    IS_FIRST_ROUND = "is_first_round"
+    MY_RANK = "my_rank"
+    INITIAL_LEARNING_RATE = "initial_learning_rate"
+    CURRENT_LEARNING_RATE = "current_learning_rate"
+    NUMBER_OF_GPUS = "number_of_gpus"
+    META_COOKIE = "cookie"
+    META_DATA = "meta_data"
+    GLOBAL_MODEL = "global_model"
+
+    IS_BEST = "is_best"
+    FAILURE = "failure"
+
+    LOG_DIR = "model_log_dir"
+    CKPT_PRELOAD_PATH = "ckpt_preload_path"
+
+    DXO = "DXO"
+
+    PHASE = "_phase_"
+    PHASE_INIT = "_init_"
+    PHASE_TRAIN = "train"
+    PHASE_MODEL_VALIDATION = "model_validation"
+    PHASE_FINISHED = "_finished_"
+
+    STATUS_WAIT = "_wait_"
+    STATUS_DONE = "_done_"
+    STATUS_TRAINING = "_training_"
+    STATUS_IDLE = "_idle_"
+
+    MODEL_LOAD_PATH = "_model_load_path"
+    MODEL_SAVE_PATH = "_model_save_path"
+    DEFAULT_MODEL_DIR = "models"
+
+    ROUND = "_round_"
+    MODEL_WEIGHTS = "_model_weights_"
+    AGGREGATION_RESULT = "_aggregation_result"
+    AGGREGATION_TRIGGERED = "_aggregation_triggered"
+    AGGREGATION_ACCEPTED = "_aggregation_accepted"
+    TRAIN_SHAREABLE = "_train_shareable_"
+    TRAINING_RESULT = "_training_result_"
+
+    SUBMIT_MODEL_FAILURE_REASON = "_submit_model_failure_reason"
+    CROSS_VAL_DIR = "cross_site_val"
+    CROSS_VAL_MODEL_DIR_NAME = "model_shareables"
+    CROSS_VAL_RESULTS_DIR_NAME = "result_shareables"
+    CROSS_VAL_MODEL_PATH = "_cross_val_model_path_"
+    CROSS_VAL_RESULTS_PATH = "_cross_val_results_path_"
+    RECEIVED_MODEL = "_receive_model_"
+    RECEIVED_MODEL_OWNER = "_receive_model_owner_"
+    MODEL_TO_VALIDATE = "_model_to_validate_"
+    DATA_CLIENT = "_data_client_"
+    VALIDATION_RESULT = "_validation_result_"
+
+    TASK_SUBMIT_MODEL = "submit_model"
+    TASK_VALIDATION = "validate"
+
+    CROSS_VAL_SERVER_MODEL = "_cross_val_server_model_"
+    CROSS_VAL_CLIENT_MODEL = "_cross_val_client_model_"
+    PARTICIPATING_CLIENTS = "_particpating_clients_"
+
+    MODEL_OWNER = "_model_owner_"
+
+    DEFAULT_FORMATTER_ID = "formatter"
+    DEFAULT_MODEL_LOCATOR_ID = "model_locator"
+
+    TASK_END_RUN = "_end_run_"
+    TASK_TRAIN = "train"
+
+    DEFAULT_AGGREGATOR_ID = "aggregator"
+    DEFAULT_PERSISTOR_ID = "persistor"
+    DEFAULT_SHAREABLE_GENERATOR_ID = "shareable_generator"
+
+
+class EnvironmentKey(object):
+
+    CHECKPOINT_DIR = "APP_CKPT_DIR"
+    CHECKPOINT_FILE_NAME = "APP_CKPT"
+
+
+class DefaultCheckpointFileName(object):
+
+    GLOBAL_MODEL = "FL_global_model.pt"
+    BEST_GLOBAL_MODEL = "best_FL_global_model.pt"
diff --git a/nvflare/app_common/app_event_type.py b/nvflare/app_common/app_event_type.py
new file mode 100644
index 0000000000..9f44f7a031
--- /dev/null
+++ b/nvflare/app_common/app_event_type.py
@@ -0,0 +1,70 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class AppEventType(object):
+    """
+    Defines application events.
+    """
+
+    START_ROUND = "_start_round"
+    END_ROUND = "_end_round"
+
+    BEFORE_AGGREGATION = "_before_aggregation"
+    END_AGGREGATION = "_end_aggregation"
+
+    SUBMIT_LOCAL_BEST_MODEL = "_submit_local_best_model"
+    SERVER_RECEIVE_BEST_MODEL = "_server_receive_best_model"
+    RECEIVE_VALIDATION_MODEL = "_receive_validation_model"
+    SEND_VALIDATION_RESULTS = "_send_validation_results"
+    RECEIVE_VALIDATION_RESULTS = "_receive_validation_results"
+
+    BEFORE_INITIALIZE = "_before_initialize"
+    AFTER_INITIALIZE = "_after_initialize"
+    BEFORE_TRAIN = "_before_train"
+    AFTER_TRAIN = "_after_train"
+
+    BEFORE_SHAREABLE_TO_LEARNABLE = "_before_model_update"
+    AFTER_SHAREABLE_TO_LEARNABLE = "_after_model_update"
+    BEFORE_LEARNABLE_PERSIST = "_before_save_model"
+    AFTER_LEARNABLE_PERSIST = "_after_save_model"
+    BEFORE_SEND_BEST_MODEL = "_before_send_best_model"
+    AFTER_SEND_BEST_MODEL = "_after_send_best_model"
+    LOCAL_BEST_MODEL_AVAILABLE = "_local_best_model_available"
+    GLOBAL_BEST_MODEL_AVAILABLE = "_global_best_model_available"
+    BEFORE_GET_VALIDATION_MODELS = "_before_get_validation_models"
+    AFTER_GET_VALIDATION_MODELS = "_after_get_validation_models"
+    SEND_MODEL_FOR_VALIDATION = "_send_model_for_validation"
+    BEFORE_VALIDATE_MODEL = "_before_validate_model"
+    AFTER_VALIDATE_MODEL = "_after_validate_model"
+    BEFORE_SUBMIT_VALIDATION_RESULTS = "_before_submit_validation_results"
+    AFTER_SUBMIT_VALIDATION_RESULTS = "_after_submit_validation_results"
+
+    # Events
+    ROUND_STARTED = "_round_started"
+    ROUND_DONE = "_round_done"
+    INITIAL_MODEL_LOADED = "_initial_model_loaded"
+    BEFORE_TRAIN_TASK = "_before_train_task"
+    RECEIVE_CONTRIBUTION = "_receive_contribution"
+    AFTER_CONTRIBUTION_ACCEPT = "_after_contribution_accept"
+    AFTER_AGGREGATION = "_after_aggregation"
+    BEFORE_CONTRIBUTION_ACCEPT = "_before_contribution_accept"
+    GLOBAL_WEIGHTS_UPDATED = "_global_weights_updated"
+    TRAINING_STARTED = "_training_started"
+    TRAINING_FINISHED = "_training_finished"
+    TRAIN_DONE = "_train_done"
+
+    CROSS_VAL_INIT = "_cross_val_init"
+    VALIDATION_RESULT_RECEIVED = "_validation_result_received"
+    RECEIVE_BEST_MODEL = "_receive_best_model"
diff --git a/nvflare/app_common/executors/__init__.py b/nvflare/app_common/executors/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/nvflare/app_common/executors/multi_process_executor.py b/nvflare/app_common/executors/multi_process_executor.py
new file mode 100644
index 0000000000..e77bc3499d
--- /dev/null
+++ b/nvflare/app_common/executors/multi_process_executor.py
@@ -0,0 +1,336 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import shlex
+import socket
+import subprocess
+import threading
+import time
+from abc import abstractmethod
+from multiprocessing.connection import Client, Listener
+
+from nvflare.apis.event_type import EventType
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.apis.signal import Signal
+from nvflare.apis.utils.fl_context_utils import get_serializable_data
+from nvflare.fuel.common.multi_process_executor_constants import CommunicateData, CommunicationMetaData
+from nvflare.fuel.utils.class_utils import ModuleScanner
+from nvflare.fuel.utils.component_builder import ComponentBuilder
+
+
+def _get_open_ports(number):
+    ports = []
+    for i in range(number):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.bind(("", 0))
+        s.listen(1)
+        port = s.getsockname()[1]
+        s.close()
+        ports.append(port)
+    return ports
+
+
+class WorkerComponentBuilder(ComponentBuilder):
+    FL_PACKAGES = ["nvflare"]
+    FL_MODULES = ["client", "app"]
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.module_scanner = ModuleScanner(WorkerComponentBuilder.FL_PACKAGES, WorkerComponentBuilder.FL_MODULES, True)
+
+    def get_module_scanner(self):
+        return self.module_scanner
+
+
+class MultiProcessExecutor(Executor):
+    def __init__(self, executor_id=None, num_of_processes=1, components=None):
+        """
+        Arguments:
+            executor_id: executor component ID
+            num_of_processes: number of processes to create
+        """
+        super().__init__()
+        self.executor_id = executor_id
+
+        self.components = {}
+        self.handlers = []
+        self._build_components(components)
+
+        if not isinstance(num_of_processes, int):
+            raise TypeError(f"{num_of_processes} must be an instance of int.")
+        if num_of_processes < 1:
+            raise ValueError(f"{num_of_processes} must >= 1.")
+        self.num_of_processes = num_of_processes
+        self.executor = None
+
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.conn_clients = []
+        self.exe_process = None
+        self.open_ports = []
+
+        self.stop_execute = False
+        self.relay_threads = []
+        self.finalized = False
+        self.event_lock = threading.Lock()
+        self.relay_lock = threading.Lock()
+
+    @abstractmethod
+    def get_multi_process_command(self) -> str:
+        """
+        Provide the command for starting multi-process execution.
+        Returns: multi-process starting command
+
+        """
+        return ""
+
+    def _build_components(self, components):
+        component_builder = WorkerComponentBuilder()
+        for item in components:
+            cid = item.get("id", None)
+            if not cid:
+                raise TypeError("missing component id")
+            self.components[cid] = component_builder.build_component(item)
+            if isinstance(self.components[cid], FLComponent):
+                self.handlers.append(self.components[cid])
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.initialize(fl_ctx)
+        elif event_type == EventType.END_RUN:
+            self.finalize(fl_ctx)
+
+        self._pass_event_to_rank_processes(event_type, fl_ctx)
+
+    def _pass_event_to_rank_processes(self, event_type: str, fl_ctx: FLContext):
+        event_site = fl_ctx.get_prop(FLContextKey.EVENT_ORIGIN_SITE)
+
+        if event_site != CommunicateData.SUB_WORKER_PROCESS:
+            with self.event_lock:
+                try:
+                    data = {
+                        CommunicationMetaData.COMMAND: CommunicateData.HANDLE_EVENT,
+                        CommunicationMetaData.FL_CTX: get_serializable_data(fl_ctx),
+                        CommunicationMetaData.EVENT_TYPE: event_type,
+                    }
+                    # send the init data to all the child processes
+                    for conn_client in self.conn_clients:
+                        conn_client[CommunicationMetaData.HANDLE_CONN].send(data)
+
+                    return_data = self.conn_clients[0][CommunicationMetaData.HANDLE_CONN].recv()
+                    # update the fl_ctx from the child process return data.
+                    fl_ctx.props.update(return_data[CommunicationMetaData.FL_CTX].props)
+                except BaseException as e:
+                    # Warning: Have to set fire_event=False, otherwise it will cause dead loop on the event handling!!!
+                    self.log_warning(
+                        fl_ctx, f"Failed to relay the event to child processes. Event: {event_type}", fire_event=False
+                    )
+
+    def initialize(self, fl_ctx: FLContext):
+        self.executor = self.components.get(self.executor_id, None)
+        if not isinstance(self.executor, Executor):
+            raise ValueError(
+                "invalid executor {}: expect Executor but got {}".format(self.executor_id, type(self.executor))
+            )
+        self._initialize_multi_process(fl_ctx)
+
+    def _initialize_multi_process(self, fl_ctx: FLContext):
+
+        try:
+            self.open_ports = _get_open_ports(self.num_of_processes * 3)
+
+            command = (
+                self.get_multi_process_command()
+                + " -m nvflare.private.fed.app.client.sub_worker_process"
+                + " -m "
+                + fl_ctx.get_prop(FLContextKey.ARGS).workspace
+                + " --ports "
+                + "-".join([str(i) for i in self.open_ports])
+            )
+            self.logger.info(f"multi_process_executor command: {command}")
+            # use os.setsid to create new process group ID
+            self.exe_process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=os.environ.copy())
+
+            for i in range(self.num_of_processes):
+                listen_port = self.open_ports[i * 3 + 2]
+                thread = threading.Thread(target=self._relay_fire_event, args=(listen_port, fl_ctx))
+                self.relay_threads.append(thread)
+                thread.start()
+
+                open_port = self.open_ports[i * 3]
+                exe_conn = self._create_connection(open_port)
+
+                open_port = self.open_ports[i * 3 + 1]
+                handle_conn = self._create_connection(open_port)
+
+                self.conn_clients.append(
+                    {CommunicationMetaData.EXE_CONN: exe_conn, CommunicationMetaData.HANDLE_CONN: handle_conn}
+                )
+            self.logger.info(f"Created the connections to child processes on ports: {str(self.open_ports)}")
+
+            data = {
+                CommunicationMetaData.FL_CTX: get_serializable_data(fl_ctx),
+                CommunicationMetaData.COMPONENTS: self.components,
+                CommunicationMetaData.HANDLERS: self.handlers,
+                CommunicationMetaData.LOCAL_EXECUTOR: self.executor,
+            }
+
+            # send the init data to all the child processes
+            responses = []
+            for conn_client in self.conn_clients:
+                conn_client[CommunicationMetaData.EXE_CONN].send(data)
+                responses.append(False)
+
+            while True:
+                run_abort_signal = fl_ctx.get_run_abort_signal()
+                if run_abort_signal and run_abort_signal.triggered:
+                    self.finalize(fl_ctx)
+                    break
+
+                # Make sure to receive responses from all rank processes.
+                index = 0
+                received_all = True
+                for conn_client in self.conn_clients:
+                    received_all = received_all and responses[index]
+                    if not responses[index]:
+                        if conn_client[CommunicationMetaData.EXE_CONN].poll(0.2):
+                            conn_client[CommunicationMetaData.EXE_CONN].recv()
+                            responses[index] = True
+                    index += 1
+                if received_all:
+                    break
+        except:
+            self.log_exception(fl_ctx, "error initializing multi_process executor")
+
+    def _relay_fire_event(self, listen_port, fl_ctx: FLContext):
+        address = ("localhost", int(listen_port))
+        listener = Listener(address, authkey=CommunicationMetaData.PARENT_PASSWORD.encode())
+        conn = listener.accept()
+
+        while not self.stop_execute:
+            try:
+                if conn.poll(0.1):
+                    data = conn.recv()
+                    event_type = data[CommunicationMetaData.EVENT_TYPE]
+                    rank_number = data[CommunicationMetaData.RANK_NUMBER]
+
+                    with self.relay_lock:
+                        fl_ctx.props.update(data[CommunicationMetaData.FL_CTX].props)
+
+                        fl_ctx.set_prop(FLContextKey.FROM_RANK_NUMBER, rank_number, private=True, sticky=False)
+                        fl_ctx.set_prop(
+                            FLContextKey.EVENT_ORIGIN_SITE,
+                            CommunicateData.SUB_WORKER_PROCESS,
+                            private=True,
+                            sticky=False,
+                        )
+                        engine = fl_ctx.get_engine()
+                        engine.fire_event(event_type, fl_ctx)
+
+                        return_data = {CommunicationMetaData.FL_CTX: get_serializable_data(fl_ctx)}
+                    conn.send(return_data)
+            except:
+                self.logger.warning("Failed to relay the fired events from rank_processes.")
+
+    def _create_connection(self, open_port):
+        conn = None
+        while not conn:
+            try:
+                address = ("localhost", open_port)
+                conn = Client(address, authkey=CommunicationMetaData.CHILD_PASSWORD.encode())
+            except BaseException as e:
+                time.sleep(1.0)
+                pass
+        return conn
+
+    def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable:
+        if not self.executor:
+            raise RuntimeError("There's no executor for task {}".format(task_name))
+
+        return self._execute_multi_process(
+            task_name=task_name, shareable=shareable, fl_ctx=fl_ctx, abort_signal=abort_signal
+        )
+
+    def _execute_multi_process(
+        self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal
+    ) -> Shareable:
+
+        if abort_signal.triggered:
+            self.finalize(fl_ctx)
+            return make_reply(ReturnCode.OK)
+
+        try:
+            data = {
+                CommunicationMetaData.COMMAND: CommunicateData.EXECUTE,
+                CommunicationMetaData.TASK_NAME: task_name,
+                CommunicationMetaData.SHAREABLE: shareable,
+                CommunicationMetaData.FL_CTX: get_serializable_data(fl_ctx),
+            }
+
+            # send the execute command to all the child processes
+            for conn_client in self.conn_clients:
+                conn_client[CommunicationMetaData.EXE_CONN].send(data)
+
+            while True:
+                if abort_signal.triggered:
+                    self.finalize(fl_ctx)
+                    return make_reply(ReturnCode.OK)
+
+                if self.conn_clients[0][CommunicationMetaData.EXE_CONN].poll(1.0):
+                    # Only need to receive the Shareable and FLContext update from rank 0 process.
+                    return_data = self.conn_clients[0][CommunicationMetaData.EXE_CONN].recv()
+                    shareable = return_data[CommunicationMetaData.SHAREABLE]
+                    fl_ctx.props.update(return_data[CommunicationMetaData.FL_CTX].props)
+                    return shareable
+        except BaseException as e:
+            self.log_error(fl_ctx, "Multi-Process Execution error.")
+            return make_reply(ReturnCode.EXECUTION_RESULT_ERROR)
+
+    def finalize(self, fl_ctx: FLContext):
+        """This is called when exiting/aborting the executor."""
+        if self.finalized:
+            return
+
+        self.finalized = True
+        self.stop_execute = True
+
+        data = {CommunicationMetaData.COMMAND: CommunicateData.CLOSE}
+        for conn_client in self.conn_clients:
+            try:
+                conn_client[CommunicationMetaData.EXE_CONN].send(data)
+                conn_client[CommunicationMetaData.HANDLE_CONN].send(data)
+                self.logger.info("close command sent to rank processes.")
+            except:
+                self.logger.warning("Failed to send the close command. ")
+        try:
+            os.killpg(os.getpgid(self.exe_process.pid), 9)
+            self.logger.debug("kill signal sent")
+        except Exception as e:
+            pass
+
+        if self.exe_process:
+            self.exe_process.terminate()
+
+        # wait for all relay threads to join!
+        for t in self.relay_threads:
+            assert isinstance(t, threading.Thread)
+            if t.is_alive():
+                t.join()
+
+        self.log_info(fl_ctx, "Multi-Process Executor finalized!", fire_event=False)
diff --git a/nvflare/app_common/filters/__init__.py b/nvflare/app_common/filters/__init__.py
new file mode 100644
index 0000000000..d20d6d1846
--- /dev/null
+++ b/nvflare/app_common/filters/__init__.py
@@ -0,0 +1,19 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .exclude_vars import ExcludeVars
+from .percentile_privacy import PercentilePrivacy
+from .svt_privacy import SVTPrivacy
+
+__all__ = ["PercentilePrivacy", "SVTPrivacy", "ExcludeVars"]
diff --git a/nvflare/app_common/filters/convert_weights.py b/nvflare/app_common/filters/convert_weights.py
new file mode 100644
index 0000000000..068cf368f3
--- /dev/null
+++ b/nvflare/app_common/filters/convert_weights.py
@@ -0,0 +1,110 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.dxo import DataKind, MetaKey, from_shareable
+from nvflare.apis.filter import Filter
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+
+class ConvertWeights(Filter):
+
+    """
+    This filter converts from
+    """
+
+    WEIGHTS_TO_DIFF = "weights_to_diff"
+    DIFF_TO_WEIGHTS = "diff_to_weights"
+
+    def __init__(self, direction: str):
+        Filter.__init__(self)
+        if direction not in (self.WEIGHTS_TO_DIFF, self.DIFF_TO_WEIGHTS):
+            raise ValueError(
+                "invalid convert direction {}: must be in {}".format(
+                    direction, (self.WEIGHTS_TO_DIFF, self.DIFF_TO_WEIGHTS)
+                )
+            )
+
+        self.direction = direction
+
+    def _get_base_weights(self, fl_ctx: FLContext):
+        task_data = fl_ctx.get_prop(FLContextKey.TASK_DATA, None)
+        if not isinstance(task_data, Shareable):
+            self.log_error(fl_ctx, "invalid task data: expect Shareable but got {}".format(type(task_data)))
+            return None
+
+        try:
+            dxo = from_shareable(task_data)
+        except ValueError:
+            self.log_error(fl_ctx, "invalid task data: no DXO")
+            return None
+
+        if dxo.data_kind != DataKind.WEIGHTS:
+            self.log_info(fl_ctx, "ignored task: expect data to be WEIGHTS but got {}".format(dxo.data_kind))
+            return None
+
+        processed_algo = dxo.get_meta_prop(MetaKey.PROCESSED_ALGORITHM, None)
+        if processed_algo:
+            self.log_info(fl_ctx, "ignored task since its processed by {}".format(processed_algo))
+            return None
+
+        return dxo.data
+
+    def process(self, shareable: Shareable, fl_ctx: FLContext) -> Shareable:
+        rc = shareable.get_return_code()
+        if rc != ReturnCode.OK:
+            # don't process if RC not OK
+            return shareable
+
+        base_weights = self._get_base_weights(fl_ctx)
+        if not base_weights:
+            return shareable
+
+        try:
+            dxo = from_shareable(shareable)
+        except ValueError:
+            self.log_error(fl_ctx, "invalid task result: no DXO")
+            return shareable
+
+        processed_algo = dxo.get_meta_prop(MetaKey.PROCESSED_ALGORITHM, None)
+        if processed_algo:
+            self.log_info(fl_ctx, "cannot process task result since its processed by {}".format(processed_algo))
+            return shareable
+
+        if self.direction == self.WEIGHTS_TO_DIFF:
+            if dxo.data_kind != DataKind.WEIGHTS:
+                self.log_warning(fl_ctx, "cannot process task result: expect WEIGHTS but got {}".format(dxo.data_kind))
+                return shareable
+
+            new_weights = dxo.data
+            for k, _ in new_weights.items():
+                if k in base_weights:
+                    new_weights[k] -= base_weights[k]
+            dxo.data_kind = DataKind.WEIGHT_DIFF
+        else:
+            # diff to weights
+            if dxo.data_kind != DataKind.WEIGHT_DIFF:
+                self.log_warning(
+                    fl_ctx, "cannot process task result: expect WEIGHT_DIFF but got {}".format(dxo.data_kind)
+                )
+                return shareable
+
+            new_weights = dxo.data
+            for k, _ in new_weights.items():
+                if k in base_weights:
+                    new_weights[k] += base_weights[k]
+            dxo.data_kind = DataKind.WEIGHTS
+
+        return dxo.update_shareable(shareable)
diff --git a/nvflare/app_common/filters/exclude_vars.py b/nvflare/app_common/filters/exclude_vars.py
new file mode 100644
index 0000000000..69da54857b
--- /dev/null
+++ b/nvflare/app_common/filters/exclude_vars.py
@@ -0,0 +1,104 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import List, Union
+
+from nvflare.apis.dxo import DXO, from_shareable
+from nvflare.apis.filter import Filter
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+
+class ExcludeVars(Filter):
+    """Exclude/Remove variables from Sharable
+
+    Args:
+        exclude_vars (Union[List[str], str, None] , optional): variables/layer names to be excluded.
+
+    Notes:
+        Based on different types of exclude_vars, this filter has different behavior:
+            if a list of variable/layer names, only specified variables will be excluded.
+            if a string, it will be converted into a regular expression, only matched variables will be excluded.
+            if not provided or other formats the Shareable remains unchanged.
+    """
+
+    def __init__(self, exclude_vars: Union[List[str], str, None] = None):
+        super().__init__()
+        self.exclude_vars = exclude_vars
+        self.skip = False
+        if self.exclude_vars is not None:
+            if not (isinstance(self.exclude_vars, list) or isinstance(self.exclude_vars, str)):
+                self.skip = True
+                self.logger.debug("Need to provide a list of layer names or a string for regex matching")
+                return
+
+            if isinstance(self.exclude_vars, list):
+                for var in self.exclude_vars:
+                    if not isinstance(var, str):
+                        self.skip = True
+                        self.logger.debug("encrypt_layers needs to be a list of layer names to encrypt.")
+                        return
+                self.logger.debug(f"Excluding {self.exclude_vars} from shareable")
+            elif isinstance(self.exclude_vars, str):
+                self.exclude_vars = re.compile(self.exclude_vars) if self.exclude_vars else None
+                if self.exclude_vars is None:
+                    self.skip = True
+                self.logger.debug(f'Excluding all layers based on regex matches with "{self.exclude_vars}"')
+        else:
+            self.logger.debug("Not excluding anything")
+            self.skip = True
+
+    def process(self, shareable: Shareable, fl_ctx: FLContext) -> Shareable:
+        if self.skip:
+            return shareable
+
+        rc = shareable.get_return_code()
+        if rc != ReturnCode.OK:
+            # don't process if RC not OK
+            return shareable
+
+        try:
+            dxo = from_shareable(shareable)
+        except:
+            self.log_exception(fl_ctx, "shareable data is not a valid DXO")
+            return shareable
+
+        assert isinstance(dxo, DXO)
+        if dxo.data is None:
+            self.log_debug(fl_ctx, "no data to filter")
+            return shareable
+
+        weights = dxo.data
+        # remove variables
+        n_excluded = 0
+        var_names = list(weights.keys())  # make a copy of keys
+        n_vars = len(var_names)
+
+        for var_name in var_names:
+            if (isinstance(self.exclude_vars, re.Pattern) and self.exclude_vars.search(var_name)) or (
+                isinstance(self.exclude_vars, list) and var_name in self.exclude_vars
+            ):
+                self.log_debug(fl_ctx, f"Excluding {var_name}")
+                weights.pop(var_name, None)
+                n_excluded += 1
+
+        if isinstance(self.exclude_vars, re.Pattern) and n_excluded == 0:
+            self.log_warning(fl_ctx, f"No matching layers found with regex {self.exclude_vars}")
+
+        self.log_debug(fl_ctx, f"Excluded {n_excluded} of {n_vars} variables. {len(weights.keys())} remaining.")
+
+        dxo.data = weights
+        return dxo.update_shareable(shareable)
diff --git a/nvflare/app_common/filters/percentile_privacy.py b/nvflare/app_common/filters/percentile_privacy.py
new file mode 100644
index 0000000000..40c0c5751b
--- /dev/null
+++ b/nvflare/app_common/filters/percentile_privacy.py
@@ -0,0 +1,102 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from nvflare.apis.dxo import DXO, MetaKey, from_shareable
+from nvflare.apis.filter import Filter
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+
+class PercentilePrivacy(Filter):
+    """
+    implementation of "largest percentile to share" privacy preserving policy
+
+    Shokri and Shmatikov, Privacy-preserving deep learning, CCS '15
+    """
+
+    def __init__(self, percentile=10, gamma=0.01):
+        super().__init__()
+
+        # must be in 0..100, only update abs diff greater than percentile
+        self.percentile = percentile
+        # must be positive
+        self.gamma = gamma  # truncate absolute value of delta W
+
+    def process(self, shareable: Shareable, fl_ctx: FLContext) -> Shareable:
+        """Compute the percentile on the abs delta_W,
+        only share the params where absolute delta_W greater than
+        the percentile value
+
+        Args:
+            shareable: information from client
+            fl_ctx: context provided by workflow
+
+        Returns:
+            Shareable
+        """
+
+        self.log_debug(fl_ctx, "inside filter")
+
+        rc = shareable.get_return_code()
+        if rc != ReturnCode.OK:
+            # don't process if RC not OK
+            return shareable
+
+        try:
+            dxo = from_shareable(shareable)
+        except:
+            self.log_exception(fl_ctx, "shareable data is not a valid DXO")
+            return shareable
+
+        assert isinstance(dxo, DXO)
+        if dxo.data is None:
+            self.log_debug(fl_ctx, "no data to filter")
+            return shareable
+
+        self.logger.debug("check gamma")
+        if self.gamma <= 0:
+            self.log_debug(fl_ctx, "no partial model: gamma: {}".format(self.gamma))
+            return shareable  # do nothing
+        if self.percentile < 0 or self.percentile > 100:
+            self.log_debug(fl_ctx, "no partial model: percentile: {}".format(self.percentile))
+            return shareable  # do nothing
+
+        # invariant to local steps
+        model_diff = dxo.data
+        total_steps = dxo.get_meta_prop(MetaKey.NUM_STEPS_CURRENT_ROUND, 1)
+
+        delta_w = {name: model_diff[name] / total_steps for name in model_diff}
+        # abs delta
+        all_abs_values = np.concatenate([np.abs(delta_w[name].ravel()) for name in delta_w])
+        cutoff = np.percentile(a=all_abs_values, q=self.percentile, overwrite_input=False)
+        self.log_info(
+            fl_ctx,
+            f"Max abs delta_w: {np.max(all_abs_values)}, Min abs delta_w: {np.min(all_abs_values)}, {cutoff=}, scale: {total_steps}.",
+        )
+
+        for name in delta_w:
+            diff_w = delta_w[name]
+            if np.ndim(diff_w) == 0:  # single scalar, no clipping
+                delta_w[name] = diff_w * total_steps
+                continue
+            selector = (diff_w > -cutoff) & (diff_w < cutoff)
+            diff_w[selector] = 0.0
+            diff_w = np.clip(diff_w, a_min=-self.gamma, a_max=self.gamma)
+            delta_w[name] = diff_w * total_steps
+
+        dxo.data = delta_w
+        return dxo.update_shareable(shareable)
diff --git a/nvflare/app_common/filters/svt_privacy.py b/nvflare/app_common/filters/svt_privacy.py
new file mode 100644
index 0000000000..79ccdace7a
--- /dev/null
+++ b/nvflare/app_common/filters/svt_privacy.py
@@ -0,0 +1,129 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from nvflare.apis.dxo import DXO, MetaKey, from_shareable
+from nvflare.apis.filter import Filter
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+
+class SVTPrivacy(Filter):
+
+    """
+    implementation of the standard SVT differential privacy algorithm.
+    """
+
+    def __init__(self, fraction=0.1, epsilon=0.1, noise_var=0.1, gamma=1e-5, tau=1e-6):
+        super().__init__()
+
+        self.frac = fraction  # fraction of the model to upload
+        self.eps_1 = epsilon
+        self.eps_2 = None  # to be derived from eps_1
+        self.eps_3 = noise_var
+        self.gamma = gamma
+        self.tau = tau
+
+    def process(self, shareable: Shareable, fl_ctx: FLContext) -> Shareable:
+        """Compute the differentially private SVT
+
+        Args:
+            shareable: information from client
+            fl_ctx: context provided by workflow
+
+        Returns:
+            Shareable
+        """
+        self.log_debug(fl_ctx, "inside filter")
+
+        rc = shareable.get_return_code()
+        if rc != ReturnCode.OK:
+            # don't process if RC not OK
+            return shareable
+
+        try:
+            dxo = from_shareable(shareable)
+        except:
+            self.log_exception(fl_ctx, "shareable data is not a valid DXO")
+            return shareable
+
+        assert isinstance(dxo, DXO)
+        if dxo.data is None:
+            self.log_debug(fl_ctx, "no data to filter")
+            return shareable
+
+        model_diff = dxo.data
+        total_steps = dxo.get_meta_prop(MetaKey.NUM_STEPS_CURRENT_ROUND, 1)
+
+        delta_w = np.concatenate([model_diff[name].ravel() / np.float(total_steps) for name in sorted(model_diff)])
+        self.log_info(
+            fl_ctx,
+            "Delta_w: Max abs: {}, Min abs: {}, Median abs: {}.".format(
+                np.max(np.abs(delta_w)), np.min(np.abs(delta_w)), np.median(np.abs(delta_w))
+            ),
+        )
+
+        # precompute thresholds
+        n_upload = np.minimum(np.ceil(np.float(delta_w.size) * self.frac), np.float(delta_w.size))
+
+        # eps_1: threshold with noise
+        lambda_rho = self.gamma * 2.0 / self.eps_1
+        threshold = self.tau + np.random.laplace(scale=lambda_rho)
+        # eps_2: query with noise
+        self.eps_2 = self.eps_1 * (2.0 * n_upload) ** (2.0 / 3.0)
+        lambda_nu = self.gamma * 4.0 * n_upload / self.eps_2
+        self.logger.info(
+            "total params: %s, epsilon: %s, "
+            "perparam budget %s, threshold tau: %s + f(eps_1) = %s, "
+            "clip gamma: %s",
+            delta_w.size,
+            self.eps_1,
+            self.eps_1 / n_upload,
+            self.tau,
+            threshold,
+            self.gamma,
+        )
+
+        # selecting weights with additive noise
+        accepted, candidate_idx = [], np.arange(delta_w.size)
+        _clipped_w = np.abs(np.clip(delta_w, a_min=-self.gamma, a_max=self.gamma))
+        while len(accepted) < n_upload:
+            nu_i = np.random.laplace(scale=lambda_nu, size=candidate_idx.shape)
+            above_threshold = (_clipped_w[candidate_idx] + nu_i) >= threshold
+            accepted += candidate_idx[above_threshold].tolist()
+            candidate_idx = candidate_idx[~above_threshold]
+            self.log_info(fl_ctx, "selected {} responses, requested {}".format(len(accepted), n_upload))
+        accepted = np.random.choice(accepted, size=np.int64(n_upload))
+        # eps_3 return with noise
+        noise = np.random.laplace(scale=self.gamma * 2.0 / self.eps_3, size=accepted.shape)
+        self.log_info(fl_ctx, "noise max: {}, median {}".format(np.max(np.abs(noise)), np.median(np.abs(noise))))
+        delta_w[accepted] = np.clip(delta_w[accepted] + noise, a_min=-self.gamma, a_max=self.gamma)
+        delta_w[candidate_idx] = 0.0
+
+        # resume original format
+        dp_w, _start = {}, 0
+        for name in sorted(model_diff):
+            if np.ndim(model_diff[name]) == 0:
+                dp_w[name] = model_diff[name]
+                _start += 1
+                continue
+            value = delta_w[_start : (_start + model_diff[name].size)]
+            dp_w[name] = value.reshape(model_diff[name].shape) * np.float(total_steps)
+            _start += model_diff[name].size
+
+        # We update the shareable weights only.  Headers are unchanged.
+        dxo.data = dp_w
+        return dxo.update_shareable(shareable)
diff --git a/nvflare/app_common/handlers/__init__.py b/nvflare/app_common/handlers/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/app_common/handlers/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/app_common/handlers/event_recorder.py b/nvflare/app_common/handlers/event_recorder.py
new file mode 100644
index 0000000000..8362195770
--- /dev/null
+++ b/nvflare/app_common/handlers/event_recorder.py
@@ -0,0 +1,369 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from typing import Dict
+
+from nvflare.apis.client_engine_spec import ClientEngineSpec
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.server_engine_spec import ServerEngineSpec
+from nvflare.apis.shareable import Shareable
+
+
+class _CtxPropReq(object):
+    """Requirements of a prop in the FLContext.
+
+    Arguments:
+        dtype: data type of the prop.
+        is_private: if this prop is private.
+        is_sticky: if this prop is sticky.
+        allow_none: if this prop can be None
+    """
+
+    def __init__(self, dtype, is_private, is_sticky, allow_none: bool = False):
+        self.dtype = dtype
+        self.is_private = is_private
+        self.is_sticky = is_sticky
+        self.allow_none = allow_none
+
+
+class _EventReq(object):
+    """Requirements for FL and peer context when an event is fired.
+
+    Arguments:
+        ctx_reqs: A dictionary that describes the requirements for fl_ctx. It maps property names to _CtxPropReq
+        peer_ctx_reqs: A dictionary that describes the requirements for peer_ctx. It maps property names to _CtxPropReq
+    """
+
+    def __init__(
+        self,
+        ctx_reqs: Dict[str, _CtxPropReq],
+        peer_ctx_reqs: Dict[str, _CtxPropReq],
+        ctx_block_list: [str] = None,
+        peer_ctx_block_list: [str] = None,
+    ):
+        self.ctx_reqs = ctx_reqs  # prop name => _CtxPropReq
+        self.peer_ctx_reqs = peer_ctx_reqs
+
+        if ctx_block_list is None:
+            ctx_block_list = []
+
+        if peer_ctx_block_list is None:
+            peer_ctx_block_list = []
+
+        self.ctx_block_list = ctx_block_list
+        self.peer_ctx_block_list = peer_ctx_block_list
+
+
+class _EventStats(object):
+    """Stats of each event."""
+
+    def __init__(self):
+        self.call_count = 0
+        self.prop_missing = 0
+        self.prop_none_value = 0
+        self.prop_dtype_mismatch = 0
+        self.prop_attr_mismatch = 0
+        self.prop_block_list_violation = 0
+        self.peer_ctx_missing = 0
+
+
+class EventRecorder(FLComponent):
+
+    _KEY_CTX_TYPE = "ctx_type"
+    _KEY_EVENT_TYPE = "event_type"
+    _KEY_EVENT_STATS = "event_stats"
+    _KEY_EVENT_REQ = "event_req"
+
+    def __init__(self, log_file_name=None):
+        super().__init__()
+
+        all_ctx_reqs = {
+            "__run_num__": _CtxPropReq(dtype=int, is_private=False, is_sticky=True),
+            "__identity_name__": _CtxPropReq(dtype=str, is_private=False, is_sticky=True),
+        }
+
+        run_req = _EventReq(ctx_reqs=all_ctx_reqs, peer_ctx_reqs={})
+        self.event_reqs = {EventType.START_RUN: run_req, EventType.END_RUN: run_req}  # event type => _EventReq
+        self.event_stats = {}  # event_type => _EventStats
+        self._log_handler_added = False
+        self.log_file_name = log_file_name if log_file_name else "event_recorded.txt"
+
+    def event_tag(self, fl_ctx: FLContext):
+        event_type = fl_ctx.get_prop(self._KEY_EVENT_TYPE, "?")
+        event_id = fl_ctx.get_prop(FLContextKey.EVENT_ID, None)
+        if event_id:
+            return "[type={}, id={}]".format(event_type, event_id)
+        else:
+            return "[{}]".format(event_type)
+
+    def event_error_tag(self, fl_ctx: FLContext):
+        ctx_type = fl_ctx.get_prop(self._KEY_CTX_TYPE, "?")
+        return "Event {}: in {},".format(self.event_tag(fl_ctx), ctx_type)
+
+    def validate_prop(self, prop_name: str, req: _CtxPropReq, fl_ctx: FLContext):
+        stats = fl_ctx.get_prop(self._KEY_EVENT_STATS, None)
+        assert isinstance(stats, _EventStats)
+
+        assert isinstance(req, _CtxPropReq)
+        detail = fl_ctx.get_prop_detail(prop_name)
+        if not isinstance(detail, dict):
+            stats.prop_missing += 1
+            self.logger.error("{} required prop '{}' doesn't exist".format(self.event_error_tag(fl_ctx), prop_name))
+            return
+
+        value = detail["value"]
+        if value is None and not req.allow_none:
+            stats.prop_none_value += 1
+            self.logger.error(
+                "{} prop '{}' is None, but None is not allowed".format(self.event_error_tag(fl_ctx), prop_name)
+            )
+
+        if req.dtype is not None:
+            if not isinstance(value, req.dtype):
+                stats.prop_dtype_mismatch += 1
+                self.logger.error(
+                    "{} prop '{}' should be {}, but got {}".format(
+                        self.event_error_tag(fl_ctx), prop_name, req.dtype, type(value)
+                    )
+                )
+
+        if req.is_private and not detail["private"]:
+            stats.prop_attr_mismatch += 1
+            self.logger.error(
+                "{} prop '{}' should be private but is public".format(self.event_error_tag(fl_ctx), prop_name)
+            )
+
+        if req.is_private is not None and not req.is_private and detail["private"]:
+            stats.prop_attr_mismatch += 1
+            self.logger.error(
+                "{} prop '{}' should be public but is private".format(self.event_error_tag(fl_ctx), prop_name)
+            )
+
+        if req.is_sticky and not detail["sticky"]:
+            stats.prop_attr_mismatch += 1
+            self.logger.error(
+                "{} prop '{}' should be sticky but is non-sticky".format(self.event_error_tag(fl_ctx), prop_name)
+            )
+
+        if req.is_sticky is not None and not req.is_sticky and detail["sticky"]:
+            stats.prop_attr_mismatch += 1
+            self.logger.error(
+                "{} prop '{}' should be non-sticky but is sticky".format(self.event_error_tag(fl_ctx), prop_name)
+            )
+
+    def check_block_list(self, block_list, fl_ctx: FLContext):
+        stats = fl_ctx.get_prop(self._KEY_EVENT_STATS, None)
+        assert isinstance(stats, _EventStats)
+        for prop_name in block_list:
+            detail = fl_ctx.get_prop_detail(prop_name)
+            if detail:
+                stats.prop_block_list_violation += 1
+                self.logger.error("{} prop {} is not expected".format(self.event_error_tag(fl_ctx), prop_name))
+
+    def check_props(self, fl_ctx: FLContext):
+        event_req = fl_ctx.get_prop(self._KEY_EVENT_REQ)
+        assert isinstance(event_req, _EventReq)
+
+        stats = fl_ctx.get_prop(self._KEY_EVENT_STATS)
+        assert isinstance(stats, _EventStats)
+
+        for prop_name, req in event_req.ctx_reqs.items():
+            self.validate_prop(prop_name, req, fl_ctx)
+
+        self.check_block_list(event_req.ctx_block_list, fl_ctx)
+
+        if event_req.peer_ctx_reqs:
+            peer_ctx = fl_ctx.get_peer_context()
+            if not peer_ctx:
+                stats.peer_ctx_missing += 1
+                self.logger.error("{} expected peer_ctx not present".format(self.event_error_tag(fl_ctx)))
+            else:
+                for prop_name, req in event_req.peer_ctx_reqs.items():
+                    self.validate_prop(prop_name, req, peer_ctx)
+                self.check_block_list(event_req.peer_ctx_block_list, peer_ctx)
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if not self._log_handler_added:
+            workspace = fl_ctx.get_engine().get_workspace()
+            app_dir = workspace.get_app_dir(fl_ctx.get_run_number())
+            output_file_handler = logging.FileHandler(os.path.join(app_dir, self.log_file_name))
+            formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+            output_file_handler.setFormatter(formatter)
+            self.logger.addHandler(output_file_handler)
+            self._log_handler_added = True
+        event_stats = self.event_stats.get(event_type, None)
+        if not event_stats:
+            event_stats = _EventStats()
+            self.event_stats[event_type] = event_stats
+
+        fl_ctx.set_prop(key=self._KEY_EVENT_STATS, value=event_stats, private=True, sticky=False)
+        fl_ctx.set_prop(key=self._KEY_EVENT_TYPE, value=event_type, private=True, sticky=False)
+        fl_ctx.set_prop(key=self._KEY_CTX_TYPE, value="fl_ctx", private=True, sticky=False)
+
+        self.log_info(fl_ctx, "Got event {}".format(self.event_tag(fl_ctx)), fire_event=False)
+        event_stats.call_count += 1
+
+        peer_ctx = fl_ctx.get_peer_context()
+        if peer_ctx:
+            event_id = fl_ctx.get_prop(FLContextKey.EVENT_ID)
+            peer_ctx.set_prop(key=FLContextKey.EVENT_ID, value=event_id, private=True, sticky=False)
+            peer_ctx.set_prop(key=self._KEY_EVENT_STATS, value=event_stats, private=True, sticky=False)
+            peer_ctx.set_prop(key=self._KEY_EVENT_TYPE, value=event_type, private=True, sticky=False)
+            peer_ctx.set_prop(key=self._KEY_CTX_TYPE, value="peer_ctx", private=True, sticky=False)
+            self.log_info(
+                fl_ctx, "Peer Context for event {}: {}".format(self.event_tag(fl_ctx), peer_ctx), fire_event=False
+            )
+
+        event_req = self.event_reqs.get(event_type, None)
+        fl_ctx.set_prop(key=self._KEY_EVENT_REQ, value=event_req, private=True, sticky=False)
+        if event_req:
+            self.check_props(fl_ctx)
+
+        if event_type == EventType.END_RUN:
+            # print stats
+            for e, s in self.event_stats.items():
+                self.log_info(fl_ctx, "Stats of {}: {}".format(e, vars(s)), fire_event=False)
+
+
+class ServerEventRecorder(EventRecorder):
+    def __init__(self):
+        super().__init__()
+
+        task_data_filter_reqs = _EventReq(
+            ctx_reqs={
+                "__engine__": _CtxPropReq(dtype=ServerEngineSpec, is_private=True, is_sticky=True),
+                FLContextKey.TASK_ID: _CtxPropReq(dtype=str, is_private=True, is_sticky=False),
+                FLContextKey.TASK_NAME: _CtxPropReq(dtype=str, is_private=True, is_sticky=False),
+                FLContextKey.TASK_DATA: _CtxPropReq(dtype=Shareable, is_private=True, is_sticky=False, allow_none=True),
+                "testPrivateServerSticky": _CtxPropReq(dtype=str, is_private=True, is_sticky=True),
+                "testPublicServerSticky": _CtxPropReq(dtype=str, is_private=False, is_sticky=True),
+            },
+            ctx_block_list=[
+                "testPrivateServerNonSticky",
+                "testPublicServerNonSticky",
+                "testPrivateClientNonSticky",
+                "testPublicClientNonSticky",
+                "testPrivateClientSticky",
+                "testPublicClientSticky",
+            ],
+            peer_ctx_reqs={
+                "__run_num__": _CtxPropReq(dtype=int, is_private=None, is_sticky=None),
+                "__identity_name__": _CtxPropReq(dtype=str, is_private=None, is_sticky=None),
+                "testPublicClientSticky": _CtxPropReq(dtype=str, is_private=None, is_sticky=None),
+            },
+            peer_ctx_block_list=[
+                "__engine__",
+                "testPrivateClientSticky",
+                "testPrivateClientNonSticky",
+                "testPublicClientNonSticky",
+            ],
+        )
+        self.event_reqs.update(
+            {
+                EventType.BEFORE_TASK_DATA_FILTER: task_data_filter_reqs,
+                EventType.AFTER_TASK_DATA_FILTER: task_data_filter_reqs,
+            }
+        )
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            fl_ctx.set_prop(
+                key="testPrivateServerSticky", value="this is a server private sticky", private=True, sticky=True
+            )
+
+            fl_ctx.set_prop(
+                key="testPublicServerSticky", value="this is a server public sticky", private=False, sticky=True
+            )
+
+            fl_ctx.set_prop(
+                key="testPrivateServerNonSticky",
+                value="this is a server private non-sticky",
+                private=True,
+                sticky=False,
+            )
+
+            fl_ctx.set_prop(
+                key="testPublicServerNonSticky", value="this is a server public non-sticky", private=False, sticky=False
+            )
+
+        super().handle_event(event_type, fl_ctx)
+
+
+class ClientEventRecorder(EventRecorder):
+    def __init__(self):
+        super().__init__()
+
+        task_data_filter_reqs = _EventReq(
+            ctx_reqs={
+                "__engine__": _CtxPropReq(dtype=ClientEngineSpec, is_private=True, is_sticky=True),
+                FLContextKey.TASK_ID: _CtxPropReq(dtype=str, is_private=True, is_sticky=False),
+                FLContextKey.TASK_NAME: _CtxPropReq(dtype=str, is_private=True, is_sticky=False),
+                FLContextKey.TASK_DATA: _CtxPropReq(dtype=Shareable, is_private=True, is_sticky=False, allow_none=True),
+                "testPrivateClientSticky": _CtxPropReq(dtype=str, is_private=True, is_sticky=True),
+                "testPublicClientSticky": _CtxPropReq(dtype=str, is_private=False, is_sticky=True),
+            },
+            ctx_block_list=[
+                "testPrivateServerNonSticky",
+                "testPublicServerNonSticky",
+                "testPrivateClientNonSticky",
+                "testPublicClientNonSticky",
+                "testPrivateServerSticky",
+                "testPublicServerSticky",
+            ],
+            peer_ctx_reqs={
+                "__run_num__": _CtxPropReq(dtype=int, is_private=None, is_sticky=None),
+                "__identity_name__": _CtxPropReq(dtype=str, is_private=None, is_sticky=None),
+                "testPublicServerSticky": _CtxPropReq(dtype=str, is_private=None, is_sticky=None),
+            },
+            peer_ctx_block_list=[
+                "__engine__",
+                "testPrivateServerSticky",
+                "testPrivateServerNonSticky",
+                "testPublicServerNonSticky",
+            ],
+        )
+        self.event_reqs.update(
+            {
+                EventType.BEFORE_TASK_DATA_FILTER: task_data_filter_reqs,
+                EventType.AFTER_TASK_DATA_FILTER: task_data_filter_reqs,
+            }
+        )
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            fl_ctx.set_prop(
+                key="testPrivateClientSticky", value="this is a client private sticky", private=True, sticky=True
+            )
+
+            fl_ctx.set_prop(
+                key="testPublicClientSticky", value="this is a client public sticky", private=False, sticky=True
+            )
+
+            fl_ctx.set_prop(
+                key="testPrivateClientNonSticky",
+                value="this is a client private non-sticky",
+                private=True,
+                sticky=False,
+            )
+
+            fl_ctx.set_prop(
+                key="testPublicClientNonSticky", value="this is a client public non-sticky", private=False, sticky=False
+            )
+
+        super().handle_event(event_type, fl_ctx)
diff --git a/nvflare/app_common/handlers/intime_model_selection_handler.py b/nvflare/app_common/handlers/intime_model_selection_handler.py
new file mode 100644
index 0000000000..9e938e13bf
--- /dev/null
+++ b/nvflare/app_common/handlers/intime_model_selection_handler.py
@@ -0,0 +1,139 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from nvflare.apis.dxo import DXO, DataKind, MetaKey, from_shareable
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey, ReservedKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.app_event_type import AppEventType
+
+
+class IntimeModelSelectionHandler(FLComponent):
+    def __init__(self, weigh_by_local_iter=False, aggregation_weights=None):
+        """
+
+        Args:
+            weigh_by_local_iter: default False
+            validation_metric_name: meta data name used for model selection
+        """
+
+        super().__init__()
+
+        self.val_metric = self.best_val_metric = -np.inf
+        self.weigh_by_local_iter = weigh_by_local_iter
+        self.validation_metric_name = MetaKey.INITIAL_METRICS
+        self.aggregation_weights = aggregation_weights or {}
+
+        self.logger.debug(f"model selection weights control: {aggregation_weights}")
+        self._reset_stats()
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        """
+            perform the handler process based on the event_type.
+
+        Args:
+            event_type (str): event type delivered from workflow
+            fl_ctx (FLContext): FL context, including peer context and other information
+
+        Returns:
+
+        """
+        if event_type == EventType.START_RUN:
+            self._startup(fl_ctx)
+        elif event_type == EventType.BEFORE_PROCESS_SUBMISSION:
+            self._before_accept(fl_ctx)
+        elif event_type == AppEventType.BEFORE_AGGREGATION:
+            self._before_aggregate(fl_ctx)
+
+    def _startup(self, fl_ctx):
+        self._reset_stats()
+
+    def _reset_stats(self):
+        self.validation_mertic_weighted_sum = 0
+        self.validation_metric_sum_of_weights = 0
+
+    def _before_accept(self, fl_ctx: FLContext):
+        peer_ctx = fl_ctx.get_peer_context()
+        shareable: Shareable = peer_ctx.get_prop(FLContextKey.SHAREABLE)
+        try:
+            dxo = from_shareable(shareable)
+        except:
+            self.log_exception(fl_ctx, "shareable data is not a valid DXO")
+            return False
+
+        assert isinstance(dxo, DXO)
+        if dxo.data_kind not in (DataKind.WEIGHT_DIFF, DataKind.WEIGHTS):
+            self.log_debug(fl_ctx, "I cannot handle {}".format(dxo.data_kind))
+            return False
+
+        if dxo.data is None:
+            self.log_debug(fl_ctx, "no data to filter")
+            return False
+
+        contribution_round = shareable.get_header(AppConstants.CONTRIBUTION_ROUND)
+        client_name = shareable.get_peer_prop(ReservedKey.IDENTITY_NAME, default="?")
+
+        current_round = fl_ctx.get_prop(AppConstants.CURRENT_ROUND)
+
+        if current_round == 0:
+            self.log_debug(fl_ctx, "skipping round 0")
+            return False  # There is no aggregated model at round 0
+
+        if contribution_round != current_round:
+            self.log_debug(
+                fl_ctx,
+                f"discarding shareable from {client_name} for round: {contribution_round}. Current round is: {current_round}",
+            )
+            return False
+
+        validation_metric = dxo.get_meta_prop(self.validation_metric_name)
+        if validation_metric is None:
+            self.log_debug(fl_ctx, f"validation metric not existing in {client_name}")
+            return False
+        else:
+            self.log_info(fl_ctx, f"validation metric {validation_metric} from client {client_name}")
+
+        if self.weigh_by_local_iter:
+            n_iter = dxo.get_meta_prop(MetaKey.NUM_STEPS_CURRENT_ROUND, 1.0)
+        else:
+            n_iter = 1.0
+
+        aggregation_weights = self.aggregation_weights.get(client_name, 1.0)
+        self.log_debug(fl_ctx, f"aggregation weight: {aggregation_weights}")
+
+        self.validation_mertic_weighted_sum += validation_metric * n_iter * aggregation_weights
+        self.validation_metric_sum_of_weights += n_iter
+        return True
+
+    def _before_aggregate(self, fl_ctx):
+        if self.validation_metric_sum_of_weights == 0:
+            self.log_debug(fl_ctx, "nothing accumulated")
+            return False
+        self.val_metric = self.validation_mertic_weighted_sum / self.validation_metric_sum_of_weights
+        self.logger.debug(f"weighted validation metric {self.val_metric}")
+        if self.val_metric > self.best_val_metric:
+            self.best_val_metric = self.val_metric
+            current_round = fl_ctx.get_prop(AppConstants.CURRENT_ROUND)
+            self.log_info(fl_ctx, f"new best validation metric at round {current_round}: {self.best_val_metric}")
+
+            # Fire event to notify that the current global model is a new best
+            self.fire_event(AppEventType.GLOBAL_BEST_MODEL_AVAILABLE, fl_ctx)
+
+        self._reset_stats()
+        return True
diff --git a/nvflare/app_common/homomorphic_encryption/__init__.py b/nvflare/app_common/homomorphic_encryption/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/app_common/homomorphic_encryption/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/app_common/homomorphic_encryption/he_constant.py b/nvflare/app_common/homomorphic_encryption/he_constant.py
new file mode 100644
index 0000000000..3a4a3ac90e
--- /dev/null
+++ b/nvflare/app_common/homomorphic_encryption/he_constant.py
@@ -0,0 +1,15 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HE_ALGORITHM_CKKS = "CKKS"
diff --git a/nvflare/app_common/homomorphic_encryption/he_intime_accumulate_model_aggregator.py b/nvflare/app_common/homomorphic_encryption/he_intime_accumulate_model_aggregator.py
new file mode 100644
index 0000000000..2d2de6acfe
--- /dev/null
+++ b/nvflare/app_common/homomorphic_encryption/he_intime_accumulate_model_aggregator.py
@@ -0,0 +1,233 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import time
+
+import numpy as np
+import tenseal as ts
+
+import nvflare.app_common.homomorphic_encryption.he_constant as he
+from nvflare.apis.dxo import DXO, DataKind, MetaKey, from_shareable
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_constant import ReservedKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.abstract.aggregator import Aggregator
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.homomorphic_encryption.homomorphic_encrypt import (
+    count_encrypted_layers,
+    load_tenseal_context_from_workspace,
+)
+
+
+class HEInTimeAccumulateWeightedAggregator(Aggregator):
+    def __init__(
+        self,
+        exclude_vars=None,
+        aggregation_weights=None,
+        tenseal_context_file="server_context.tenseal",
+        weigh_by_local_iter=False,
+        expected_data_kind="WEIGHT_DIFF",
+        expected_algorithm=he.HE_ALGORITHM_CKKS,
+    ):
+        """
+            In time aggregator for `Shareables` encrypted using homomorphic encryption (HE) with TenSEAL https://github.com/OpenMined/TenSEAL.
+
+        Args:
+            exclude_vars: variable names that should be excluded from aggregation (use regular expression)
+            aggregation_weights: dictionary of client aggregation `{"client1": 1.0, "client2": 2.0, "client3": 3.0}`;
+                                 defaults to a weight of 1.0 if not specified. Will be ignored if weigh_by_local_iter: False (default for HE)
+            weigh_by_local_iter: If true, multiply client weights on first in encryption space
+                                 (default: `False` which is recommended for HE, first multiply happens in `HEModelEncryptor`)
+        """
+        super().__init__()
+        self.tenseal_context = None
+        self.tenseal_context_file = tenseal_context_file
+        if expected_data_kind not in [DataKind.WEIGHT_DIFF, DataKind.WEIGHTS]:
+            raise ValueError(f"{expected_data_kind=} not in WEIGHT_DIFF or WEIGHTS")
+        self.expected_data_kind = expected_data_kind
+        self.expected_algorithm = expected_algorithm
+        if self.expected_algorithm != he.HE_ALGORITHM_CKKS:
+            raise ValueError(f"expected algorithm {self.expected_algorithm} not supported")
+        self.exclude_vars = re.compile(exclude_vars) if exclude_vars else None
+        self.aggregation_weights = aggregation_weights or {}
+        self.reset_stats()
+        self.weigh_by_local_iter = weigh_by_local_iter
+        self.logger.info(f"client weights control: {self.aggregation_weights}")
+        if not self.weigh_by_local_iter:
+            if self.aggregation_weights:
+                self.logger.warning("aggregation_weights will be ignored if weigh_by_local_iter=False")
+            self.logger.info("Only divide by sum of local (weighted) iterations.")
+        self.warning_count = dict()
+        self.warning_limit = 0
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.tenseal_context = load_tenseal_context_from_workspace(self.tenseal_context_file, fl_ctx)
+        elif event_type == EventType.END_RUN:
+            self.tenseal_context = None
+
+    def reset_stats(self):
+        self.total = dict()
+        self.counts = dict()
+        self.contribution_count = 0
+        self.history = list()
+        self.merged_encrypted_layers = dict()  # thread-safety is handled by workflow
+
+    def accept(self, shareable: Shareable, fl_ctx: FLContext) -> bool:
+        """
+        Accepts and adds the client updates to current average in HE encrypted space.
+
+        Args:
+            shareable:
+            fl_ctx:
+
+        Returns:
+            The first boolean indicates if this shareable is accepted.
+            The second bollean indicates if aggregate can be called.
+        """
+        dxo = from_shareable(shareable)
+        if dxo.data_kind != DataKind.WEIGHT_DIFF:
+            self.log_error(fl_ctx, "support WEIGHT_DIFF type DXO only, skipping this shareable.")
+            return False
+
+        enc_algo = dxo.get_meta_prop(key=MetaKey.PROCESSED_ALGORITHM, default=None)
+        if enc_algo != self.expected_algorithm:
+            self.log_error(fl_ctx, "unsupported encryption algorithm {enc_algo}")
+            return False
+
+        current_round = fl_ctx.get_prop(AppConstants.CURRENT_ROUND)
+        client_name = shareable.get_peer_prop(ReservedKey.IDENTITY_NAME, "?")
+        contribution_round = shareable.get_header(AppConstants.CONTRIBUTION_ROUND)
+
+        rc = shareable.get_return_code()
+        if rc and rc != ReturnCode.OK:
+            self.log_debug(fl_ctx, f"Client {client_name} returned rc: {rc}. Disregarding contribution.")
+            return False
+
+        self.log_debug(fl_ctx, f"current_round: {current_round}")
+
+        if contribution_round != current_round:
+            self.log_debug(
+                fl_ctx,
+                "Discarded the contribution from {client_name} for round: {contribution_round}. Current round is: {current_round}",
+            )
+            return False
+
+        start_time = time.time()
+
+        for item in self.history:
+            if client_name == item["client_name"]:
+                prev_round = item["round"]
+                self.log_info(
+                    fl_ctx,
+                    f"discarding shareable from {client_name} at round: {contribution_round} as {prev_round} accepted already",
+                )
+                return False
+
+        self.log_info(fl_ctx, f"Adding contribution from {client_name}.")
+
+        n_iter = dxo.get_meta_prop(key=MetaKey.NUM_STEPS_CURRENT_ROUND)
+        if n_iter is None:
+            if self.warning_count.get(client_name, 0) <= self.warning_limit:
+                self.log_warning(
+                    fl_ctx,
+                    f"NUM_STEPS_CURRENT_ROUND missing"
+                    f" from {client_name} and set to default value, 1.0. "
+                    f" This kind of message will show {self.warning_limit} times at most.",
+                )
+                if client_name in self.warning_count:
+                    self.warning_count[client_name] = self.warning_count[client_name] + 1
+                else:
+                    self.warning_count[client_name] = 0
+            n_iter = 1.0
+        float_n_iter = np.float(n_iter)
+
+        aggregation_weight = self.aggregation_weights.get(client_name)
+        if aggregation_weight is None:
+            aggregation_weight = 1.0
+
+        aggr_data = dxo.data
+        encrypted_layers = dxo.get_meta_prop(MetaKey.PROCESSED_KEYS)
+        # TODO: test support of different encrypted layers for different clients!
+
+        if encrypted_layers is None:
+            self.log_error(fl_ctx, "encrypted_layers is None!")
+            return False
+
+        for k, v in aggr_data.items():
+            if self.exclude_vars is not None and self.exclude_vars.search(k):
+                continue
+            if encrypted_layers[k]:
+                if self.weigh_by_local_iter:
+                    weighted_value = ts.ckks_vector_from(self.tenseal_context, v) * (aggregation_weight * float_n_iter)
+                else:
+                    weighted_value = ts.ckks_vector_from(self.tenseal_context, v)
+                self.merged_encrypted_layers[k] = True  # any client can set this true
+            else:
+                if self.weigh_by_local_iter:
+                    weighted_value = v * (aggregation_weight * float_n_iter)
+                else:
+                    weighted_value = v
+                if k not in self.merged_encrypted_layers:
+                    self.merged_encrypted_layers[k] = False  # only set False if no other client set it to True
+            current_total = self.total.get(k, None)
+            if current_total is None:
+                self.total[k] = weighted_value
+                self.counts[k] = n_iter
+            else:
+                self.total[k] = current_total + weighted_value
+                self.counts[k] = self.counts[k] + n_iter
+
+        self.contribution_count += 1
+
+        end_time = time.time()
+        n_encrypted, n_total = count_encrypted_layers(self.merged_encrypted_layers)
+        self.log_info(fl_ctx, f"{n_encrypted} of {n_total} layers encrypted")
+        self.log_info(fl_ctx, f"Round {current_round} adding {client_name} time is {end_time - start_time} seconds")
+
+        self.history.append(
+            {
+                "client_name": client_name,
+                "round": contribution_round,
+                "aggregation_weight": aggregation_weight,
+                "n_iter": n_iter,
+            }
+        )
+        return True
+
+    def aggregate(self, fl_ctx: FLContext) -> Shareable:
+        start_time = time.time()
+        current_round = fl_ctx.get_prop(AppConstants.CURRENT_ROUND)
+
+        aggregated_dict = dict()
+        for k, v in self.total.items():
+            aggregated_dict[k] = v * (1.0 / self.counts[k])
+        end_time = time.time()
+        self.log_info(
+            fl_ctx,
+            f"Aggregated {self.contribution_count} contributions for round {current_round} time is {end_time - start_time} seconds",
+        )
+
+        dxo = DXO(data_kind=self.expected_data_kind, data=aggregated_dict)
+        dxo.set_meta_prop(MetaKey.PROCESSED_KEYS, self.merged_encrypted_layers)
+        dxo.set_meta_prop(MetaKey.PROCESSED_ALGORITHM, self.expected_algorithm)
+        n_encrypted, n_total = count_encrypted_layers(self.merged_encrypted_layers)
+        self.log_info(fl_ctx, f"{n_encrypted} of {n_total} layers encrypted")
+
+        fl_ctx.set_prop(AppConstants.DXO, dxo, private=True, sticky=False)
+
+        self.reset_stats()  # only reset dictionary after adding merged_encrypted_layers to dictionary
+        return dxo.to_shareable()
diff --git a/nvflare/app_common/homomorphic_encryption/he_model_decryptor.py b/nvflare/app_common/homomorphic_encryption/he_model_decryptor.py
new file mode 100644
index 0000000000..8b2e6ff9ee
--- /dev/null
+++ b/nvflare/app_common/homomorphic_encryption/he_model_decryptor.py
@@ -0,0 +1,157 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+import numpy as np
+import tenseal as ts
+from tenseal.tensors.ckksvector import CKKSVector
+
+import nvflare.app_common.homomorphic_encryption.he_constant as he
+from nvflare.apis.dxo import MetaKey, from_shareable
+from nvflare.apis.event_type import EventType
+from nvflare.apis.filter import Filter
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.homomorphic_encryption.homomorphic_encrypt import (
+    count_encrypted_layers,
+    load_tenseal_context_from_workspace,
+)
+
+
+class HEModelDecryptor(Filter):
+    def __init__(self, tenseal_context_file="client_context.tenseal"):
+        """
+            filter to decrypt Shareable object using homomorphic encryption (HE) with TenSEAL https://github.com/OpenMined/TenSEAL.
+
+        Args:
+            tenseal_context_file: tenseal context files containing decryption keys and parameters
+
+        """
+        super().__init__()
+        self.logger.info("Using HE model decryptor.")
+        self.tenseal_context = None
+        self.tenseal_context_file = tenseal_context_file
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.tenseal_context = load_tenseal_context_from_workspace(self.tenseal_context_file, fl_ctx)
+        elif event_type == EventType.END_RUN:
+            self.tenseal_context = None
+
+    def decryption(self, params, encrypted_layers, fl_ctx: FLContext):
+
+        n_params = len(params.keys())
+        self.log_info(fl_ctx, f"Running HE Decryption algorithm {n_params} variables")
+        if encrypted_layers is None:
+            raise ValueError("encrypted_layers is None!")
+
+        start_time = time.time()
+        n_decrypted, n_total = 0, 0
+        for i, param_name in enumerate(params.keys()):
+            values = params[param_name]
+            if encrypted_layers[param_name]:
+                _n = values.size()
+                n_total += _n
+                if isinstance(values, CKKSVector):
+                    self.log_info(fl_ctx, f"Decrypting vars {i+1} of {n_params}: {param_name} with {_n} values")
+                    params[param_name] = values.decrypt()
+                    n_decrypted += _n
+                else:
+                    self.log_info(
+                        fl_ctx,
+                        f"{i} of {n_params}: {param_name} = {np.shape(params[param_name])} already decrypted (RAW)!",
+                    )
+                    raise ValueError("Should be encrypted at this point!")
+            else:
+                params[param_name] = values
+        end_time = time.time()
+        self.log_info(fl_ctx, f"Decryption time for {n_decrypted} of {n_total} params {end_time - start_time} seconds.")
+
+        return params
+
+    def to_ckks_vector(self, params, encrypted_layers, fl_ctx: FLContext):
+        """convert encrypted arrays to CKKS vector"""
+        if encrypted_layers is None:
+            raise ValueError("encrypted_layers is None!")
+        start_time = time.time()
+        result = {}
+        n_total = 0
+        self.log_info(fl_ctx, f"params {len(params)} {type(params)}")
+        for v in params:
+            ndarray = params[v]
+            if encrypted_layers[v]:
+                if np.size(ndarray) > 1:
+                    raise ValueError(f"size of {v} should not be larger 1 but is {np.size(ndarray)}!")
+                result[v] = ts.ckks_vector_from(self.tenseal_context, ndarray)
+                n = result[v].size()
+            else:
+                result[v] = ndarray
+                n = np.size(ndarray)
+            n_total += n
+        end_time = time.time()
+        self.log_info(fl_ctx, f"to_ckks_vector time for {n_total} values: {end_time - start_time} seconds.")
+        return result
+
+    def process(self, shareable: Shareable, fl_ctx: FLContext) -> Shareable:
+        """
+            filter process apply to the Shareable object.
+
+        Args:
+            shareable: shareable
+            fl_ctx: FLContext
+
+        Returns:
+            a Shareable object with decrypted model weights
+
+        """
+        rc = shareable.get_return_code()
+        if rc != ReturnCode.OK:
+            # don't process if RC not OK
+            return shareable
+
+        try:
+            return self._process(shareable, fl_ctx)
+        except BaseException as e:
+            self.log_exception(fl_ctx, "error performing HE decryption")
+            raise ValueError(f"HEModelDecryptor Exception {e}")
+
+    def _process(self, shareable: Shareable, fl_ctx: FLContext):
+        self.log_info(fl_ctx, "Running decryption...")
+        dxo = from_shareable(shareable)
+
+        encrypted_algo = dxo.get_meta_prop(key=MetaKey.PROCESSED_ALGORITHM, default=None)
+        if encrypted_algo != he.HE_ALGORITHM_CKKS:
+            self.log_error(fl_ctx, "shareable is not HE CKKS encrypted")
+            return shareable
+
+        encrypted_layers = dxo.get_meta_prop(key=MetaKey.PROCESSED_KEYS, default=None)
+        if not encrypted_layers:
+            self.log_warning(fl_ctx, "dxo does not contain PROCESSED_KEYS (do nothing)")
+            return shareable
+
+        n_encrypted, n_total = count_encrypted_layers(encrypted_layers)
+        self.log_info(fl_ctx, f"{n_encrypted} of {n_total} layers encrypted")
+        decrypted_params = self.decryption(
+            params=self.to_ckks_vector(params=dxo.data, encrypted_layers=encrypted_layers, fl_ctx=fl_ctx),
+            encrypted_layers=encrypted_layers,
+            fl_ctx=fl_ctx,
+        )
+
+        dxo.data = decrypted_params
+        dxo.remove_meta_props([MetaKey.PROCESSED_ALGORITHM, MetaKey.PROCESSED_KEYS])
+        dxo.update_shareable(shareable)
+
+        return shareable
diff --git a/nvflare/app_common/homomorphic_encryption/he_model_encryptor.py b/nvflare/app_common/homomorphic_encryption/he_model_encryptor.py
new file mode 100644
index 0000000000..0de1af7f03
--- /dev/null
+++ b/nvflare/app_common/homomorphic_encryption/he_model_encryptor.py
@@ -0,0 +1,195 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import time
+
+import numpy as np
+import tenseal as ts
+from tenseal.tensors.ckksvector import CKKSVector
+
+import nvflare.app_common.homomorphic_encryption.he_constant as he
+from nvflare.apis.dxo import DXO, MetaKey, from_shareable
+from nvflare.apis.event_type import EventType
+from nvflare.apis.filter import Filter
+from nvflare.apis.fl_constant import ReservedKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.homomorphic_encryption.homomorphic_encrypt import (
+    count_encrypted_layers,
+    load_tenseal_context_from_workspace,
+)
+
+
+class HEModelEncryptor(Filter):
+    def __init__(
+        self,
+        tenseal_context_file="client_context.tenseal",
+        encrypt_layers=None,
+        aggregation_weights=None,
+        weigh_by_local_iter=True,
+    ):
+        """
+            filter to encrypt Shareable object using homomorphic encryption (HE) with TenSEAL https://github.com/OpenMined/TenSEAL.
+
+        Args:
+            tenseal_context_file: tenseal context files containing encryption keys and parameters
+            encrypt_layers: if not specified (None), all layers are being encrypted;
+                            if list of variable/layer names, only specified variables are encrypted;
+                            if string containing regular expression (e.g. "conv"), only matched variables are being encrypted.
+            aggregation_weights: dictionary of client aggregation `{"client1": 1.0, "client2": 2.0, "client3": 3.0}`;
+                                 defaults to a weight of 1.0 if not specified.
+            weigh_by_local_iter: If true, multiply client weights on first before encryption (default: `True` which is recommended for HE)
+
+        """
+        super().__init__()
+        self.logger.info("Using HE model encryptor.")
+        self.tenseal_context = None
+        self.tenseal_context_file = tenseal_context_file
+        self.aggregation_weights = aggregation_weights or {}
+        self.logger.info(f"client weights control: {self.aggregation_weights}")
+        self.weigh_by_local_iter = weigh_by_local_iter
+        self.n_iter = None
+        self.client_name = None
+        self.aggregation_weight = None
+
+        # choose which layers to encrypt
+        if encrypt_layers is not None:
+            if not (isinstance(encrypt_layers, list) or isinstance(encrypt_layers, str)):
+                raise ValueError("Provide a list of layer names or a string for regex matching")
+        if isinstance(encrypt_layers, list):
+            for encrypt_layer in encrypt_layers:
+                if not isinstance(encrypt_layer, str):
+                    raise ValueError("encrypt_layers needs to be a list of layer names to encrypt.")
+            self.encrypt_layers = encrypt_layers
+            self.logger.info(f"Encrypting {len(encrypt_layers)} layers")
+        elif isinstance(encrypt_layers, str):
+            self.encrypt_layers = re.compile(encrypt_layers) if encrypt_layers else None
+            self.logger.info(f'Encrypting all layers based on regex matches with "{encrypt_layers}"')
+        else:
+            self.encrypt_layers = [True]  # needs to be list for logic in encryption()
+            self.logger.info("Encrypting all layers")
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.tenseal_context = load_tenseal_context_from_workspace(self.tenseal_context_file, fl_ctx)
+        elif event_type == EventType.END_RUN:
+            self.tenseal_context = None
+
+    def encryption(self, params, fl_ctx: FLContext):
+        n_params = len(params.keys())
+        self.log_info(fl_ctx, f"Running HE Encryption algorithm on {n_params} variables")
+
+        # parse regex encrypt layers
+        if isinstance(self.encrypt_layers, re.Pattern):
+            re_pattern = self.encrypt_layers
+            self.encrypt_layers = []
+            for var_name in params:
+                if re_pattern.search(var_name):
+                    self.encrypt_layers.append(var_name)
+            self.log_info(fl_ctx, f"Regex found {self.encrypt_layers} matching layers.")
+            if len(self.encrypt_layers) == 0:
+                raise ValueError(f"No matching layers found with regex {re_pattern}")
+
+        start_time = time.time()
+        n_encrypted, n_total = 0, 0
+        encryption_dict = {}
+        vmins, vmaxs = [], []
+        for i, param_name in enumerate(params.keys()):
+            values = params[param_name].ravel()
+            _n = np.size(values)
+            n_total += _n
+
+            # weigh before encryption
+            if self.aggregation_weight:
+                values = values * np.float(self.aggregation_weight)
+            if self.weigh_by_local_iter:
+                values = values * np.float(self.n_iter)
+
+            if param_name in self.encrypt_layers or self.encrypt_layers[0] is True:
+                self.log_info(fl_ctx, f"Encrypting vars {i+1} of {n_params}: {param_name} with {_n} values")
+                vmin = np.min(params[param_name])
+                vmax = np.max(params[param_name])
+                vmins.append(vmin)
+                vmaxs.append(vmax)
+                params[param_name] = ts.ckks_vector(self.tenseal_context, values).serialize()
+                encryption_dict[param_name] = True
+                n_encrypted += _n
+            elif isinstance(values, CKKSVector):
+                self.log_error(
+                    fl_ctx, f"{i} of {n_params}: {param_name} = {np.shape(params[param_name])} already encrypted!"
+                )
+                raise ValueError("This should not happen!")
+            else:
+                params[param_name] = values
+                encryption_dict[param_name] = False
+        end_time = time.time()
+        if n_encrypted == 0:
+            raise ValueError("Nothing has been encrypted! Check provided encrypt_layers list of layer names or regex.")
+        self.log_info(
+            fl_ctx,
+            f"Encryption time for {n_encrypted} of {n_total} params"
+            f" (encrypted value range [{np.min(vmins)}, {np.max(vmaxs)}])"
+            f" {end_time - start_time} seconds.",
+        )
+        # params is a dictionary.  keys are layer names.  values are either weights or serialized ckks_vector of weights.
+        # encryption_dict: keys are layer names.  values are True for serialized ckks_vectors, False elsewhere.
+        return params, encryption_dict
+
+    def process(self, shareable: Shareable, fl_ctx: FLContext) -> Shareable:
+        """
+            filter process apply to the Shareable object.
+
+        Args:
+            shareable: shareable
+            fl_ctx: FLContext
+
+        Returns:
+            a Shareable object with encrypted model weights
+
+        """
+        rc = shareable.get_return_code()
+        if rc != ReturnCode.OK:
+            # don't process if RC not OK
+            return shareable
+
+        dxo = from_shareable(shareable)
+
+        if self.aggregation_weights:
+            self.client_name = shareable.get_peer_prop(ReservedKey.IDENTITY_NAME, default="?")
+            self.aggregation_weight = self.aggregation_weights.get(self.client_name, 1.0)
+            self.log_info(fl_ctx, f"weighting {self.client_name} by aggregation weight {self.aggregation_weight}")
+
+        if self.weigh_by_local_iter:
+            self.n_iter = dxo.get_meta_prop(MetaKey.NUM_STEPS_CURRENT_ROUND, None)
+            if self.n_iter is None:
+                raise ValueError("DXO data does not have local iterations for weighting!")
+            self.log_info(fl_ctx, f"weighting by local iter before encryption with {self.n_iter}")
+        try:
+            new_dxo = self._process(dxo, fl_ctx)
+            new_dxo.update_shareable(shareable)
+        except BaseException as e:
+            self.log_exception(fl_ctx, "Exception occurred")
+
+        return shareable
+
+    def _process(self, dxo: DXO, fl_ctx: FLContext) -> DXO:
+        self.log_info(fl_ctx, "Running HE encryption...")
+        encrypted_params, encryption_dict = self.encryption(params=dxo.data, fl_ctx=fl_ctx)
+        new_dxo = DXO(data_kind=dxo.data_kind, data=encrypted_params, meta=dxo.meta)
+        new_dxo.set_meta_prop(key=MetaKey.PROCESSED_KEYS, value=encryption_dict)
+        new_dxo.set_meta_prop(key=MetaKey.PROCESSED_ALGORITHM, value=he.HE_ALGORITHM_CKKS)
+        n_encrypted, n_total = count_encrypted_layers(encryption_dict)
+        self.log_info(fl_ctx, f"{n_encrypted} of {n_total} layers encrypted")
+        return new_dxo
diff --git a/nvflare/app_common/homomorphic_encryption/he_model_shareable_generator.py b/nvflare/app_common/homomorphic_encryption/he_model_shareable_generator.py
new file mode 100644
index 0000000000..d9a0113dd4
--- /dev/null
+++ b/nvflare/app_common/homomorphic_encryption/he_model_shareable_generator.py
@@ -0,0 +1,151 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+import numpy as np
+import tenseal as ts
+
+import nvflare.app_common.homomorphic_encryption.he_constant as he
+from nvflare.apis.dxo import DataKind, MetaKey, from_shareable
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.abstract.model import ModelLearnable, ModelLearnableKey
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.homomorphic_encryption.homomorphic_encrypt import load_tenseal_context_from_workspace
+from nvflare.app_common.shareablegenerators.full_model_shareable_generator import FullModelShareableGenerator
+
+
+class HEModelShareableGenerator(FullModelShareableGenerator):
+    def __init__(self, tenseal_context_file="server_context.tenseal"):
+        """
+            ShareableGenerator converts between Shareable and Learnable objects with homomorphic encryption (HE)
+            support using TenSEAL https://github.com/OpenMined/TenSEAL.
+
+        Args:
+            tenseal_context_file: tenseal context files containing decryption keys and parameters
+
+        """
+        super().__init__()
+        self.tenseal_context = None
+        self.tenseal_context_file = tenseal_context_file
+        self.is_encrypted = False
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.tenseal_context = load_tenseal_context_from_workspace(self.tenseal_context_file, fl_ctx)
+        elif event_type == EventType.END_RUN:
+            self.tenseal_context = None
+
+    def add_to_global_weights(self, fl_ctx: FLContext, new_val, base_weights, v_name, encrypt_layers):
+        if encrypt_layers is None:
+            raise ValueError("encrypted layers info missing!")
+
+        current_round = fl_ctx.get_prop(AppConstants.CURRENT_ROUND)
+        start_round = fl_ctx.get_prop(AppConstants.START_ROUND, 0)
+        if current_round > start_round and encrypt_layers[v_name]:
+            if encrypt_layers.get(v_name, False):
+                try:
+                    binary_global_var = base_weights[v_name]
+                    global_var = ts.ckks_vector_from(
+                        self.tenseal_context, binary_global_var
+                    )  # now the global model weights are encrypted
+                    n_vars_total = global_var.size()
+                except BaseException as e:
+                    raise ValueError("add_to_global_weights Exception", str(e))
+        else:
+            global_var = base_weights[v_name].ravel()
+            n_vars_total = np.size(global_var)
+
+        # update the global model
+        updated_vars = new_val + global_var
+
+        if encrypt_layers[v_name]:  # only works with standard aggregation
+            self.log_info(fl_ctx, f"serialize encrypted {v_name}")
+            updated_vars = updated_vars.serialize()
+
+        return updated_vars, n_vars_total
+
+    def _shareable_to_learnable(self, shareable: Shareable, fl_ctx: FLContext) -> ModelLearnable:
+        dxo = from_shareable(shareable)
+        enc_algorithm = dxo.get_meta_prop(MetaKey.PROCESSED_ALGORITHM)
+        if enc_algorithm != he.HE_ALGORITHM_CKKS:
+            raise ValueError("expected encryption algorithm {} but got {}".format(he.HE_ALGORITHM_CKKS, enc_algorithm))
+
+        encrypt_layers = dxo.get_meta_prop(MetaKey.PROCESSED_KEYS)
+        if encrypt_layers is None:
+            raise ValueError("DXO in shareable missing PROCESSED_KEYS property")
+
+        if len(encrypt_layers) == 0:
+            raise ValueError(f"encrypt_layers is empty: {encrypt_layers}")
+
+        base_model = fl_ctx.get_prop(AppConstants.GLOBAL_MODEL)
+        if not base_model:
+            self.system_panic(reason="No global base model!", fl_ctx=fl_ctx)
+            return base_model
+
+        base_weights = base_model[ModelLearnableKey.WEIGHTS]
+
+        if dxo.data_kind == DataKind.WEIGHT_DIFF:
+            start_time = time.time()
+            model_diff = dxo.data
+            if not model_diff:
+                raise ValueError(f"{self._name} DXO data is empty!")
+
+            n_vars = len(model_diff.items())
+            n_params = 0
+            for v_name, v_value in model_diff.items():
+                self.log_debug(fl_ctx, f"adding {v_name} to global model...")
+                # v_value += model[v_name]
+                updated_vars, n_vars_total = self.add_to_global_weights(
+                    fl_ctx, v_value, base_weights, v_name, encrypt_layers
+                )
+                n_params += n_vars_total
+                base_weights[v_name] = updated_vars
+                self.log_debug(fl_ctx, f"assigned new {v_name}")
+
+            end_time = time.time()
+            self.log_info(
+                fl_ctx,
+                f"Updated global model {n_vars} vars with {n_params} params in {end_time - start_time} seconds",
+            )
+        elif dxo.data_kind == DataKind.WEIGHTS:
+            weights = dxo.data
+            base_model[ModelLearnableKey.WEIGHTS] = weights
+        else:
+            raise NotImplementedError(f"data type {dxo.data_kind} not supported!")
+
+        self.log_debug(fl_ctx, "returning model")
+        base_model[ModelLearnableKey.META] = dxo.get_meta_props()
+        return base_model
+
+    def shareable_to_learnable(self, shareable: Shareable, fl_ctx: FLContext) -> ModelLearnable:
+        """
+            Updates the global model in `Learnable` in encrypted space.
+
+        Args:
+            shareable: shareable
+            fl_ctx: FLContext
+
+        Returns:
+            Learnable object
+
+        """
+        self.log_info(fl_ctx, "shareable_to_learnable...")
+        try:
+            return self._shareable_to_learnable(shareable, fl_ctx)
+        except BaseException as e:
+            self.log_exception(fl_ctx, "error converting shareable to model")
+            raise ValueError(f"{self._name} Exception {e}")
diff --git a/nvflare/app_common/homomorphic_encryption/he_pt_model_reader_writer.py b/nvflare/app_common/homomorphic_encryption/he_pt_model_reader_writer.py
new file mode 100644
index 0000000000..9a7ddbe275
--- /dev/null
+++ b/nvflare/app_common/homomorphic_encryption/he_pt_model_reader_writer.py
@@ -0,0 +1,59 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.pt.pt_fed_utils import feed_vars
+from nvflare.app_common.pt.pt_model_reader_writer import PTModelReaderWriter
+
+
+class HEPTModelReaderWriter(PTModelReaderWriter):
+    def apply_model(self, model_params, fl_ctx: FLContext, options=None):
+        """
+            Write global model back to local model.
+            Needed to extract local parameter shape to reshape decrypted vectors.
+
+        Args:
+        :param model_params: a ModelData message
+        :return: True if the local model changed
+        """
+        try:
+            # net = self.fitter.net
+            net = fl_ctx.get_prop(AppConstants.MODEL_NETWORK)
+            # if self.fitter.multi_gpu:
+            if fl_ctx.get_prop(AppConstants.MULTI_GPU):
+                net = net.module
+
+            # reshape decrypted parameters
+            local_var_dict = net.state_dict()
+            for var_name in local_var_dict:
+                if var_name in model_params:
+                    try:
+                        self.logger.debug(
+                            f"Reshaping {var_name}: {np.shape(model_params[var_name])} to"
+                            f" {local_var_dict[var_name].shape}",
+                        )
+                        model_params[var_name] = np.reshape(model_params[var_name], local_var_dict[var_name].shape)
+                    except Exception as e:
+                        raise RuntimeError(f"{self._name} reshaping Exception: {str(e)}")
+
+            assign_ops, updated_local_model = feed_vars(net, model_params)
+            self.logger.debug(f"assign_ops: {len(assign_ops)}")
+            self.logger.debug(f"updated_local_model: {len(updated_local_model)}")
+            net.load_state_dict(updated_local_model)
+            return assign_ops
+        except BaseException as e:
+            raise RuntimeError(f"{self._name} apply_model Exception: {str(e)}")
diff --git a/nvflare/app_common/homomorphic_encryption/homomorphic_encrypt.py b/nvflare/app_common/homomorphic_encryption/homomorphic_encrypt.py
new file mode 100644
index 0000000000..534020caf1
--- /dev/null
+++ b/nvflare/app_common/homomorphic_encryption/homomorphic_encrypt.py
@@ -0,0 +1,58 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tenseal as ts
+
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.fuel.sec.security_content_service import LoadResult, SecurityContentService
+
+
+def load_tenseal_context_from_workspace(ctx_file_name: str, fl_ctx: FLContext):
+    """
+    Loads homomorphic encryption (HE) context from TenSEAL (https://github.com/OpenMined/TenSEAL) containing encryption keys and parameters.
+
+    Args:
+        ctx_file_name: filepath of TensSEAL context file
+        fl_ctx:
+
+    Returns:
+        TenSEAL context
+
+    """
+    is_secure_mode = fl_ctx.get_prop(FLContextKey.SECURE_MODE, True)
+    data, rc = SecurityContentService.load_content(ctx_file_name)
+
+    bad_rcs = [LoadResult.INVALID_CONTENT, LoadResult.NO_SUCH_CONTENT]
+    if is_secure_mode:
+        bad_rcs.extend([LoadResult.INVALID_SIGNATURE, LoadResult.NOT_SIGNED])
+
+    if rc in bad_rcs:
+        raise ValueError("Cannot load tenseal_context {}: {}".format(ctx_file_name, rc))
+
+    context = ts.context_from(data)
+    return context
+
+
+def count_encrypted_layers(encrypted_layers: dict):
+    """
+    Count number of encrypted layers homomorphic encryption (HE) layers/variables.
+
+    """
+    n_total = len(encrypted_layers)
+    n_encrypted = 0
+    for e in encrypted_layers.keys():
+        if encrypted_layers[e]:
+            n_encrypted += 1
+    return n_encrypted, n_total
diff --git a/nvflare/app_common/pt/__init__.py b/nvflare/app_common/pt/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/app_common/pt/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/app_common/pt/pt_fed_utils.py b/nvflare/app_common/pt/pt_fed_utils.py
new file mode 100644
index 0000000000..9dd87a40c0
--- /dev/null
+++ b/nvflare/app_common/pt/pt_fed_utils.py
@@ -0,0 +1,167 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from collections import OrderedDict
+
+import torch
+import torch.nn as nn
+
+from nvflare.apis.dxo import MetaKey
+from nvflare.app_common.abstract.model import (
+    ModelLearnable,
+    ModelLearnableKey,
+    make_model_learnable,
+    validate_model_learnable,
+)
+
+
+def feed_vars(model: nn.Module, model_params):
+    """feed variable values from model_params to pytorch state_dict.
+
+    Args:
+        model (nn.Module): the local pytorch model
+        model_params: a ModelData message
+
+    Returns:
+        a list of params and a dictionary of vars to params
+    """
+    _logger = logging.getLogger("AssignVariables")
+    _logger.debug("AssignVariables...")
+
+    to_assign = []
+    n_ext = len(model_params)
+    _logger.debug(f"n_ext {n_ext}")
+
+    local_var_dict = model.state_dict()
+    for var_name in local_var_dict:
+        try:
+            if var_name in tuple(model_params):
+                nd = model_params[var_name]
+                to_assign.append(nd)
+                local_var_dict[var_name] = torch.as_tensor(
+                    nd
+                )  # update local state dict TODO: enable setting of datatype
+        except Exception as e:
+            print("pt_feed_vars Exception:", str(e))
+            raise RuntimeError(str(e))
+
+    _logger.debug("Updated local variables to be assigned.")
+
+    n_assign = len(to_assign)
+    _logger.info(f"Vars {n_ext} of {n_assign} assigned.")
+    return to_assign, local_var_dict
+
+
+class PTModelPersistenceFormatManager(object):
+
+    PERSISTENCE_KEY_MODEL = "model"
+    PERSISTENCE_KEY_TRAIN_CONF = "train_conf"
+    PERSISTENCE_KEY_META_PROPS = "meta_props"
+
+    def __init__(self, data: dict, default_train_conf=None):
+        if not isinstance(data, dict):
+            raise TypeError("data must be a dict but got {}".format(type(data)))
+
+        self.var_dict = None
+        self.meta = None
+        self.train_conf = None
+        self.other_props = {}  # other props from the original data that need to be kept
+
+        if self.PERSISTENCE_KEY_MODEL not in data:
+            # this is a simple weight dict
+            self.var_dict = data
+        else:
+            # dict of dicts
+            self.var_dict = data[self.PERSISTENCE_KEY_MODEL]
+            self.meta = data.get(self.PERSISTENCE_KEY_META_PROPS, None)
+            self.train_conf = data.get(self.PERSISTENCE_KEY_TRAIN_CONF, None)
+
+            # we need to keep other props, if any, so they can be kept when persisted
+            for k, v in data.items():
+                if k not in [
+                    self.PERSISTENCE_KEY_MODEL,
+                    self.PERSISTENCE_KEY_META_PROPS,
+                    self.PERSISTENCE_KEY_TRAIN_CONF,
+                ]:
+                    self.other_props[k] = v
+
+        if not self.train_conf:
+            self.train_conf = default_train_conf
+
+    def _get_processed_vars(self) -> dict:
+        if self.meta:
+            return self.meta.get(MetaKey.PROCESSED_KEYS, {})
+        else:
+            return {}
+
+    def to_model_learnable(self, exclude_vars) -> ModelLearnable:
+        processed_vars = self._get_processed_vars()
+
+        weights = {}
+        for k, v in self.var_dict.items():
+            if exclude_vars and exclude_vars.search(k):
+                continue
+
+            is_processed = processed_vars.get(k, False)
+            if is_processed:
+                weights[k] = v
+            else:
+                weights[k] = v.cpu().numpy()
+
+        return make_model_learnable(weights, self.meta)
+
+    def to_persistence_dict(self) -> dict:
+        processed_vars = self._get_processed_vars()
+        weights_dict = OrderedDict()
+        for k, v in self.var_dict.items():
+            is_processed = processed_vars.get(k, False)
+            if is_processed:
+                weights_dict[k] = v
+            else:
+                weights_dict[k] = torch.as_tensor(v)
+
+        # always use complex format for saving
+        persistence_dict = OrderedDict()
+        persistence_dict[self.PERSISTENCE_KEY_MODEL] = weights_dict
+        if self.meta:
+            persistence_dict[self.PERSISTENCE_KEY_META_PROPS] = self.meta
+        if self.train_conf:
+            persistence_dict[self.PERSISTENCE_KEY_TRAIN_CONF] = self.train_conf
+        if self.other_props:
+            for k, v in self.other_props.items():
+                persistence_dict[k] = v
+        return persistence_dict
+
+    def update(self, ml: ModelLearnable):
+        """
+        Update the persistence data with the learned values
+
+        Args:
+            ml:
+
+        Returns:
+
+        """
+
+        err = validate_model_learnable(ml)
+        if err:
+            raise ValueError(err)
+        self.meta = ml.get(ModelLearnableKey.META, None)
+
+        # update with value of the model learnable
+        # note that the original weights that are not learned are still kept!
+        learned_weights = ml.get(ModelLearnableKey.WEIGHTS, {})
+        for k, v in learned_weights.items():
+            self.var_dict[k] = v
diff --git a/nvflare/app_common/pt/pt_fedopt.py b/nvflare/app_common/pt/pt_fedopt.py
new file mode 100644
index 0000000000..f8b4be323e
--- /dev/null
+++ b/nvflare/app_common/pt/pt_fedopt.py
@@ -0,0 +1,227 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+import torch
+
+from nvflare.apis.dxo import DataKind, MetaKey, from_shareable
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.abstract.learnable import Learnable
+from nvflare.app_common.abstract.model import make_model_learnable
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.shareablegenerators.full_model_shareable_generator import FullModelShareableGenerator
+
+
+class PTFedOptModelShareableGenerator(FullModelShareableGenerator):
+    """ShareableGenerator to implement the FedOpt algorithm proposed in
+    Reddi, Sashank, et al. "Adaptive federated optimization." arXiv preprint arXiv:2003.00295 (2020).
+    This SharableGenerator will update the global model using the specified
+    PyTorch optimizer and learning rate scheduler.
+
+    Args:
+        optimizer_args: dictionary of optimizer arguments, e.g.
+            {'path': 'torch.optim.SGD', 'args': {'lr': 1.0}} (default).
+        lr_scheduler_args: dictionary of server-side learning rate scheduler arguments, e.g.
+            {'path': 'torch.optim.lr_scheduler.CosineAnnealingLR', 'args': {'T_max': 100}} (default: None).
+        source_model: either a valid torch model object or a component ID of a torch model object
+        device: specify the device to run server-side optimization, e.g. "cpu" or "cuda:0"
+            (will default to cuda if available and no device is specified).
+
+    Todo:
+        * write server-side lr to tensorboard
+    """
+
+    def __init__(
+        self,
+        optimizer_args: dict = None,
+        lr_scheduler_args: dict = None,
+        source_model="model",
+        device=None,
+    ):
+        super().__init__()
+        if not optimizer_args:
+            self.logger("No optimizer_args provided. Using FedOpt with SGD and lr 1.0")
+            optimizer_args = {"name": "SGD", "args": {"lr": 1.0}}
+
+        if not isinstance(optimizer_args, dict):
+            raise TypeError(
+                "optimizer_args must be a dict of format, e.g. {'path': 'torch.optim.SGD', 'args': {'lr': 1.0}}."
+            )
+        if lr_scheduler_args is not None:
+            if not isinstance(lr_scheduler_args, dict):
+                raise TypeError(
+                    "optimizer_args must be a dict of format, e.g. "
+                    "{'path': 'torch.optim.lr_scheduler.CosineAnnealingLR', 'args': {'T_max': 100}}."
+                )
+        self.source_model = source_model
+        self.optimizer_args = optimizer_args
+        self.lr_scheduler_args = lr_scheduler_args
+        self.model = None
+        self.optimizer = None
+        self.lr_scheduler = None
+        if device is None:
+            self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+        else:
+            self.device = torch.device(device)
+        self.optimizer_name = None
+        self.lr_scheduler_name = None
+
+    def _get_component_name(self, component_args):
+        if component_args is not None:
+            name = component_args.get("path", None)
+            if name is None:
+                name = component_args.get("name", None)
+            return name
+        else:
+            return None
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            # Initialize the optimizer with current global model params
+            engine = fl_ctx.get_engine()
+
+            if isinstance(self.source_model, str):
+                self.model = engine.get_component(self.source_model)
+            else:
+                self.model = self.source_model
+
+            if self.model is None:
+                self.system_panic(
+                    "Model is not available",
+                    fl_ctx,
+                )
+                return
+            elif not isinstance(self.model, torch.nn.Module):
+                self.system_panic(
+                    f"Expected model to be a torch.nn.Module but got {type(self.model)}",
+                    fl_ctx,
+                )
+                return
+            else:
+                print("server model", self.model)
+
+            self.model.to(self.device)
+
+            # set up optimizer
+            try:
+                # use provided or default optimizer arguments and add the model parameters
+                if "args" not in self.optimizer_args:
+                    self.optimizer_args["args"] = {}
+                self.optimizer_args["args"]["params"] = self.model.parameters()
+                self.optimizer = engine.build_component(self.optimizer_args)
+                # get optimizer name for log
+                self.optimizer_name = self._get_component_name(self.optimizer_args)
+            except BaseException as e:
+                self.system_panic(
+                    f"Exception while parsing `optimizer_args`: " f"{self.optimizer_args} with Exception {e}",
+                    fl_ctx,
+                )
+                return
+
+            # set up lr scheduler
+            if self.lr_scheduler_args is not None:
+                try:
+                    self.lr_scheduler_name = self._get_component_name(self.lr_scheduler_args)
+                    # use provided or default lr scheduler argument and add the optimizer
+                    if "args" not in self.lr_scheduler_args:
+                        self.lr_scheduler_args["args"] = {}
+                    self.lr_scheduler_args["args"]["optimizer"] = self.optimizer
+                    self.lr_scheduler = engine.build_component(self.lr_scheduler_args)
+                except BaseException as e:
+                    self.system_panic(
+                        f"Exception while parsing `lr_scheduler_args`: " f"{self.lr_scheduler_args} with Exception {e}",
+                        fl_ctx,
+                    )
+                    return
+
+    def server_update(self, model_diff):
+        """Updates the global model using the specified optimizer.
+
+        Args:
+            model_diff: the aggregated model differences from clients.
+
+        Returns:
+            The updated PyTorch model state dictionary.
+
+        """
+        self.model.train()
+        self.optimizer.zero_grad()
+
+        # Apply the update to the model. We must multiply weights_delta by -1.0 to
+        # view it as a gradient that should be applied to the server_optimizer.
+        for name, param in self.model.named_parameters():
+            param.grad = torch.tensor(-1.0 * model_diff[name]).to(self.device)
+
+        self.optimizer.step()
+        if self.lr_scheduler is not None:
+            self.lr_scheduler.step()
+
+        return self.model.state_dict()
+
+    def shareable_to_learnable(self, shareable: Shareable, fl_ctx: FLContext) -> Learnable:
+        """Convert Shareable to Learnable while doing a FedOpt update step
+
+        Supporting data_kind == DataKind.WEIGHT_DIFF
+
+        Args:
+            shareable (Shareable): Shareable to be converted
+            fl_ctx (FLContext): FL context
+
+        Returns:
+            Model: Updated global ModelLearnable.
+        """
+
+        # check types
+        dxo = from_shareable(shareable)
+
+        if dxo.data_kind != DataKind.WEIGHT_DIFF:
+            self.system_panic(
+                "FedOpt is only implemented for " "data_kind == DataKind.WEIGHT_DIFF",
+                fl_ctx,
+            )
+            return Learnable()
+
+        if (processed_algorithm := dxo.get_meta_prop(MetaKey.PROCESSED_ALGORITHM)) is not None:
+            self.system_panic(
+                f"FedOpt is not implemented for shareable processed by {processed_algorithm}",
+                fl_ctx,
+            )
+            return Learnable()
+
+        model_diff = dxo.data
+
+        start = time.time()
+        weights = self.server_update(model_diff)
+        secs = time.time() - start
+
+        # convert to numpy dict of weights
+        start = time.time()
+        for key in weights:
+            weights[key] = weights[key].detach().cpu().numpy()
+        secs_detach = time.time() - start
+
+        self.log_info(
+            fl_ctx,
+            f"FedOpt ({self.optimizer_name}, {self.device}) server model update "
+            f"round {fl_ctx.get_prop(AppConstants.CURRENT_ROUND)}, "
+            f"{self.lr_scheduler_name if self.lr_scheduler_name else ''} "
+            f"lr: {self.optimizer.param_groups[-1]['lr']}, "
+            f"update: {secs} secs., detach: {secs_detach} secs.",
+        )
+        # TODO: write server-side lr to tensorboard
+
+        return make_model_learnable(weights, dxo.get_meta_props())
diff --git a/nvflare/app_common/pt/pt_fedproxloss.py b/nvflare/app_common/pt/pt_fedproxloss.py
new file mode 100644
index 0000000000..7cd377d88b
--- /dev/null
+++ b/nvflare/app_common/pt/pt_fedproxloss.py
@@ -0,0 +1,43 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from torch.nn.modules.loss import _Loss
+
+
+class PTFedProxLoss(_Loss):
+    """Compute FedProx loss: a loss penalizing the deviation from global model
+
+    Args:
+        mu: weighting parameter
+        input (nn.Module): the local pytorch model
+        target (nn.Module): the copy of global pytorch model when local clients received it
+                            at the beginning of each local round
+
+    Returns:
+        FedProx loss term
+    """
+
+    def __init__(self, mu: float = 0.01) -> None:
+        super().__init__()
+        if mu < 0.0:
+            raise ValueError("mu should be no less than 0.0")
+        self.mu = mu
+
+    def forward(self, input, target) -> torch.Tensor:
+        prox_loss: torch.Tensor = 0.0
+        for param, ref in zip(input.named_parameters(), target.named_parameters()):
+            prox_loss += (self.mu / 2) * torch.sum((param[1] - ref[1]) ** 2)
+
+        return prox_loss
diff --git a/nvflare/app_common/pt/pt_file_model_persistor.py b/nvflare/app_common/pt/pt_file_model_persistor.py
new file mode 100644
index 0000000000..f3919ae869
--- /dev/null
+++ b/nvflare/app_common/pt/pt_file_model_persistor.py
@@ -0,0 +1,228 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+from collections import OrderedDict
+
+import torch
+
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from nvflare.app_common.app_constant import AppConstants, DefaultCheckpointFileName, EnvironmentKey
+from nvflare.app_common.app_event_type import AppEventType
+from nvflare.app_common.pt.pt_fed_utils import PTModelPersistenceFormatManager
+
+
+class PTFileModelPersistor(ModelPersistor):
+
+    """
+    This Model Persistor tries to load PT model data in following three ways:
+
+    1. Load from a specified source checkpoint file
+    2. Load from a location from the app folder
+    3. Load from a torch model object
+
+    The Persistor tries method 1 first if the source_ckpt_file_full_name is specified;
+    If source_ckpt_file_full_name is not specified, it tries the method 2;
+    If no checkpoint location is specified in the app folder, it tries method 3.
+
+    Method 2 - Load from a location from the app folder
+    It is assumed that the app folder must contain the environments.json file. Among other things, this
+    JSON file must specify where to find the checkpoint file. It does so with two JSON elements:
+
+        APP_CKPT_DIR - this element specifies the folder (within the app) where the checkpoint file resides.
+        APP_CKPT - this element specified the base file name of the checkpoint
+
+    Here is an example of the environments.json content:
+
+        {
+            "APP_CKPT_DIR": "model",
+            "APP_CKPT": "pretrained_model.pt"
+        }
+
+    In this example, the checkpoint file is located in the "model" folder within the app and is named
+    pretrained_model.pt.
+
+    Method 3 - Load from a torch model object. In this case, the 'model' arg must be a valid torch
+    model, or the component ID of a valid torch model included in the "components" section of
+    your config_fed_server.json.
+
+    If all 3 methods fail, system_panic() is called.
+
+    If checkpoint folder name is specified, then global model and best global model will be saved to it;
+    Otherwise they will be saved directly in the app folder.
+
+    """
+
+    def __init__(
+        self,
+        exclude_vars=None,
+        model=None,
+        global_model_file_name=DefaultCheckpointFileName.GLOBAL_MODEL,
+        best_global_model_file_name=DefaultCheckpointFileName.BEST_GLOBAL_MODEL,
+        source_ckpt_file_full_name=None,
+    ):
+        """
+
+        Args:
+            exclude_vars: regex expression specifying weight vars to be excluded from training; optional
+            model: torch model object or component id of the model object; optional
+            global_model_file_name: file name for saving global model
+            best_global_model_file_name: file name for saving best global model
+            source_ckpt_file_full_name: full file name for source model checkpoint file
+
+        """
+        super().__init__()
+        self.exclude_vars = re.compile(exclude_vars) if exclude_vars else None
+        self.model = model
+        self.log_dir = None
+        self.ckpt_preload_path = None
+        self.persistence_manager = None
+        self.ckpt_dir_env_key = EnvironmentKey.CHECKPOINT_DIR
+        self.ckpt_file_name_env_key = EnvironmentKey.CHECKPOINT_FILE_NAME
+        self.global_model_file_name = global_model_file_name
+        self.best_global_model_file_name = best_global_model_file_name
+        self.source_ckpt_file_full_name = source_ckpt_file_full_name
+
+        if source_ckpt_file_full_name and not os.path.exists(source_ckpt_file_full_name):
+            raise ValueError("specified source checkpoint model file {} does not exist")
+
+    def _initialize(self, fl_ctx: FLContext):
+        app_root = fl_ctx.get_prop(FLContextKey.APP_ROOT)
+        env = None
+        run_args = fl_ctx.get_prop(FLContextKey.ARGS)
+        if run_args:
+            env_config_file_name = os.path.join(app_root, run_args.env)
+            if os.path.exists(env_config_file_name):
+                try:
+                    with open(env_config_file_name) as file:
+                        env = json.load(file)
+                except:
+                    self.system_panic(
+                        reason="error opening env config file {}".format(env_config_file_name), fl_ctx=fl_ctx
+                    )
+                    return
+
+        if env is not None:
+            if env.get(self.ckpt_dir_env_key, None):
+                fl_ctx.set_prop(AppConstants.LOG_DIR, env[self.ckpt_dir_env_key], private=True, sticky=True)
+            if env.get(self.ckpt_file_name_env_key) is not None:
+                fl_ctx.set_prop(
+                    AppConstants.CKPT_PRELOAD_PATH, env[self.ckpt_file_name_env_key], private=True, sticky=True
+                )
+
+        log_dir = fl_ctx.get_prop(AppConstants.LOG_DIR)
+        if log_dir:
+            self.log_dir = os.path.join(app_root, log_dir)
+        else:
+            self.log_dir = app_root
+
+        self._ckpt_save_path = os.path.join(self.log_dir, self.global_model_file_name)
+        self._best_ckpt_save_path = os.path.join(self.log_dir, self.best_global_model_file_name)
+
+        ckpt_preload_path = fl_ctx.get_prop(AppConstants.CKPT_PRELOAD_PATH)
+        if ckpt_preload_path:
+            self.ckpt_preload_path = os.path.join(app_root, ckpt_preload_path)
+
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
+
+        if isinstance(self.model, str):
+            # treat it as model component ID
+            model_component_id = self.model
+            engine = fl_ctx.get_engine()
+            self.model = engine.get_component(model_component_id)
+            if not self.model:
+                self.system_panic(reason="cannot find model component '{}'".format(model_component_id), fl_ctx=fl_ctx)
+                return
+            if not isinstance(self.model, torch.nn.Module):
+                self.system_panic(
+                    reason="expect model component '{}' to be torch.nn.Module but got {}".format(
+                        model_component_id, type(self.model)
+                    ),
+                    fl_ctx=fl_ctx,
+                )
+                return
+        elif self.model and not isinstance(self.model, torch.nn.Module):
+            self.system_panic(
+                reason="expect model to be torch.nn.Module but got {}".format(type(self.model)), fl_ctx=fl_ctx
+            )
+            return
+
+        fl_ctx.sync_sticky()
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        """Convert initialised model into Learnable/Model format.
+
+        Args:
+            fl_ctx (FLContext): FL Context delivered by workflow
+
+        Returns:
+            Model: a Learnable/Model object
+        """
+
+        src_file_name = None
+        if self.source_ckpt_file_full_name:
+            src_file_name = self.source_ckpt_file_full_name
+        elif self.ckpt_preload_path:
+            src_file_name = self.ckpt_preload_path
+
+        if src_file_name:
+            try:
+                device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+                data = torch.load(src_file_name, map_location=device)
+                # "checkpoint may contain 'model', 'optimizer', 'lr_scheduler', etc. or only contain model dict directly."
+            except:
+                self.log_exception(fl_ctx, "error loading checkpoint from {}".format(src_file_name))
+                self.system_panic(reason="cannot load model checkpoint", fl_ctx=fl_ctx)
+                return None
+        else:
+            # if no pretrained model provided, use the generated network weights from APP config
+            # note that, if set "determinism" in the config, the init model weights will always be the same
+            try:
+                data = self.model.state_dict() if self.model is not None else OrderedDict()
+            except:
+                self.log_exception(fl_ctx, "error getting state_dict from model object")
+                self.system_panic(reason="cannot create state_dict from model object", fl_ctx=fl_ctx)
+                return None
+
+        if self.model:
+            default_train_conf = {"train": {"model": type(self.model).__name__}}
+        else:
+            default_train_conf = None
+
+        self.persistence_manager = PTModelPersistenceFormatManager(data, default_train_conf=default_train_conf)
+        return self.persistence_manager.to_model_learnable(self.exclude_vars)
+
+    def handle_event(self, event: str, fl_ctx: FLContext):
+        if event == EventType.START_RUN:
+            self._initialize(fl_ctx)
+        elif event == AppEventType.GLOBAL_BEST_MODEL_AVAILABLE:
+            # save the current model as the best model!
+            self.save_model_file(self._best_ckpt_save_path)
+
+    def save_model_file(self, save_path: str):
+        assert isinstance(self.persistence_manager, PTModelPersistenceFormatManager)
+        save_dict = self.persistence_manager.to_persistence_dict()
+        torch.save(save_dict, save_path)
+
+    def save_model(self, ml: ModelLearnable, fl_ctx: FLContext):
+        assert isinstance(self.persistence_manager, PTModelPersistenceFormatManager)
+        self.persistence_manager.update(ml)
+        self.save_model_file(self._ckpt_save_path)
diff --git a/nvflare/app_common/pt/pt_model_reader_writer.py b/nvflare/app_common/pt/pt_model_reader_writer.py
new file mode 100644
index 0000000000..63a6d6a734
--- /dev/null
+++ b/nvflare/app_common/pt/pt_model_reader_writer.py
@@ -0,0 +1,105 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from pathlib import Path
+
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model_processor import ModelProcessor
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.pt.pt_fed_utils import feed_vars
+
+
+class PTModelReaderWriter(ModelProcessor):
+    def __init__(self):
+        self._name = self.__class__.__name__
+        self.logger = logging.getLogger(self._name)
+
+    def initialize(self, trainer):
+        pass
+
+    def extract_model(self, model_vars, fl_ctx: FLContext):
+        # net = self.fitter.net
+        net = fl_ctx.get_prop(AppConstants.MODEL_NETWORK)
+        # if self.fitter.multi_gpu:
+        if fl_ctx.get_prop(AppConstants.MULTI_GPU):
+            net = net.module
+        local_state_dict = net.state_dict()
+
+        self.logger.debug("setup local_model_dict")
+        local_model_dict = {}
+        for var_name in local_state_dict:
+            try:
+                local_model_dict[var_name] = local_state_dict[var_name].cpu().numpy()
+            except Exception as e:
+                raise ValueError("Did not work:", str(e))
+        self.logger.debug(f"local_model_dict {len(local_model_dict)}")
+
+        return local_model_dict
+
+    def apply_model(self, model_params, fl_ctx: FLContext, options=None):
+        """Set the local model according to model_data
+
+        Args:
+            model_params: model data information
+            fl_ctx (FLContext): FL Context delivered by workflow
+            options: . Defaults to None.
+
+        Raises:
+            RuntimeError: Raised when being unable to apply model_params to the network
+
+        Returns:
+            a list of ops applied to model
+        """
+        try:
+            # net = self.fitter.net
+            net = fl_ctx.get_prop(AppConstants.MODEL_NETWORK)
+            # if self.fitter.multi_gpu:
+            if fl_ctx.get_prop(AppConstants.MULTI_GPU):
+                net = net.module
+            assign_ops, updated_local_model = feed_vars(net, model_params)
+            self.logger.debug(f"assign_ops: {len(assign_ops)}")
+            self.logger.debug(f"updated_local_model: {len(updated_local_model)}")
+            # self.fitter.net.load_state_dict(updated_local_model)
+            net.load_state_dict(updated_local_model)
+            return assign_ops
+        except Exception as e:
+            raise RuntimeError("load_state_dict Exception:", str(e))
+
+    def get_local_models(self, model_params=None):
+        """Get the local models
+
+        Args:
+            model_params: model data information
+
+        Returns:
+            a dictionary from filename to bytes
+        """
+        if not model_params or "model_log_dir" not in model_params or "model_name" not in model_params:
+            return
+
+        model_dir = Path(model_params["model_log_dir"])
+        model_name = model_params["model_name"]
+
+        # Find all checkpoint files
+        model_file = Path(model_dir, model_name)
+        if not model_file.exists():
+            return None
+
+        # Read checkpoint files into bytes
+        buffer_dict = {}
+        with model_file.open("rb") as file_bytes:
+            buffer_dict[model_file.name] = file_bytes.read()
+
+        return buffer_dict
diff --git a/nvflare/app_common/pt/pt_multi_process_executor.py b/nvflare/app_common/pt/pt_multi_process_executor.py
new file mode 100644
index 0000000000..86b17c5eed
--- /dev/null
+++ b/nvflare/app_common/pt/pt_multi_process_executor.py
@@ -0,0 +1,30 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from nvflare.app_common.executors.multi_process_executor import MultiProcessExecutor
+
+
+class PTMultiProcessExecutor(MultiProcessExecutor):
+    def __init__(self, executor_id=None, num_of_processes=1, components=None):
+        super().__init__(executor_id, num_of_processes, components)
+
+    def get_multi_process_command(self) -> str:
+        return (
+            f"{sys.executable} -m torch.distributed.run --nproc_per_node="
+            + str(self.num_of_processes)
+            + " --nnodes=1 --node_rank=0"
+            + ' --master_addr="localhost" --master_port=1234'
+        )
diff --git a/nvflare/app_common/pt/tb_receiver.py b/nvflare/app_common/pt/tb_receiver.py
new file mode 100644
index 0000000000..4f7a71576f
--- /dev/null
+++ b/nvflare/app_common/pt/tb_receiver.py
@@ -0,0 +1,83 @@
+import os
+
+from torch.utils.tensorboard import SummaryWriter
+
+from nvflare.apis.analytix import Data as AnalyticsData
+from nvflare.apis.analytix import DataType as AnalyticsDataType
+from nvflare.apis.dxo import from_shareable
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.widgets.streaming import AnalyticsReceiver
+
+FUNCTION_MAPPING = {
+    AnalyticsDataType.SCALAR: "add_scalar",
+    AnalyticsDataType.TEXT: "add_text",
+    AnalyticsDataType.IMAGE: "add_image",
+    AnalyticsDataType.SCALARS: "add_scalars",
+}
+
+
+class TBAnalyticsReceiver(AnalyticsReceiver):
+    """
+    Folder structure:
+
+    #  inside run_XX folder
+    # - workspace
+    #    - run_01 (already created):
+    #        - output_dir (default: tb_events):
+    #           - peer_name_1:
+    #           - peer_name_2:
+    #
+    #    - run_02 (already created):
+    #        - output_dir (default: tb_events):
+    #           - peer_name_1:
+    #           - peer_name_2:
+    #
+    """
+
+    def __init__(self, tb_folder="tb_events", events=None):
+        super().__init__(events=events)
+        self.writer = {}
+        self.tb_folder = tb_folder
+        self.root_log_dir = None
+
+    def initialize(self, fl_ctx: FLContext):
+        workspace = fl_ctx.get_engine().get_workspace()
+        run_dir = workspace.get_run_dir(fl_ctx.get_run_number())
+        root_log_dir = os.path.join(run_dir, self.tb_folder)
+        os.makedirs(root_log_dir, exist_ok=True)
+        self.root_log_dir = root_log_dir
+
+    def save(self, fl_ctx: FLContext, shareable: Shareable, record_origin):
+        dxo = from_shareable(shareable)
+        analytic_data = AnalyticsData.from_dxo(dxo)
+
+        writer = self.writer.get(record_origin)
+        if writer is None:
+            peer_log_dir = os.path.join(self.root_log_dir, record_origin)
+            writer = SummaryWriter(log_dir=peer_log_dir)
+            self.writer[record_origin] = writer
+
+        # depend on the type in dxo do different things
+        for k, v in dxo.data.items():
+            tag_name = f"{k}"
+            self.log_debug(
+                fl_ctx,
+                f"save tag {tag_name} and value {v} with type {analytic_data.data_type} from {record_origin}",
+                fire_event=False,
+            )
+            func_name = FUNCTION_MAPPING.get(analytic_data.data_type, None)
+            if func_name is None:
+                self.log_error(fl_ctx, f"The data_type {analytic_data.data_type} is not supported.", fire_event=False)
+                continue
+
+            func = getattr(writer, func_name)
+            if isinstance(analytic_data.kwargs, dict):
+                func(tag_name, v, **analytic_data.kwargs)
+            else:
+                func(tag_name, v)
+
+    def finalize(self, fl_ctx: FLContext):
+        for writer in self.writer.values():
+            writer.flush()
+            writer.close()
diff --git a/nvflare/app_common/shareablegenerators/__init__.py b/nvflare/app_common/shareablegenerators/__init__.py
new file mode 100644
index 0000000000..9ff9bcb074
--- /dev/null
+++ b/nvflare/app_common/shareablegenerators/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .full_model_shareable_generator import FullModelShareableGenerator
+
+__all__ = ["FullModelShareableGenerator"]
diff --git a/nvflare/app_common/shareablegenerators/full_model_shareable_generator.py b/nvflare/app_common/shareablegenerators/full_model_shareable_generator.py
new file mode 100644
index 0000000000..a0020ee501
--- /dev/null
+++ b/nvflare/app_common/shareablegenerators/full_model_shareable_generator.py
@@ -0,0 +1,69 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.dxo import DataKind, from_shareable
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.app_common.abstract.model import ModelLearnable, ModelLearnableKey, model_learnable_to_dxo
+from nvflare.app_common.abstract.shareable_generator import ShareableGenerator
+from nvflare.app_common.app_constant import AppConstants
+
+
+class FullModelShareableGenerator(ShareableGenerator):
+    def learnable_to_shareable(self, ml: ModelLearnable, fl_ctx: FLContext) -> Shareable:
+        """Convert Learnable to Shareable
+
+        Args:
+            model (Learnable): model to be converted
+            fl_ctx (FLContext): FL context
+
+        Returns:
+            Shareable: a shareable containing a DXO object,
+        """
+        dxo = model_learnable_to_dxo(ml)
+        return dxo.to_shareable()
+
+    def shareable_to_learnable(self, shareable: Shareable, fl_ctx: FLContext) -> ModelLearnable:
+        """Convert Shareable to Learnable
+
+        Supporting TYPE == TYPE_WEIGHT_DIFF or TYPE_WEIGHTS
+
+        Args:
+            shareable (Shareable): Shareable that contains a DXO object
+            fl_ctx (FLContext): FL context
+
+        Returns: a ModelLearnable object
+        """
+        base_model = fl_ctx.get_prop(AppConstants.GLOBAL_MODEL)
+        if not base_model:
+            self.system_panic(reason="No global base model!", fl_ctx=fl_ctx)
+            return base_model
+
+        weights = base_model[ModelLearnableKey.WEIGHTS]
+        dxo = from_shareable(shareable)
+
+        if dxo.data_kind == DataKind.WEIGHT_DIFF:
+            if dxo.data is not None:
+                model_diff = dxo.data
+                for v_name, v_value in model_diff.items():
+                    weights[v_name] += v_value
+        elif dxo.data_kind == DataKind.WEIGHTS:
+            weights = dxo.data
+            if not weights:
+                self.log_info(fl_ctx, "No model weights found. Model will not be updated.")
+            else:
+                base_model[ModelLearnableKey.WEIGHTS] = weights
+
+        base_model[ModelLearnableKey.META] = dxo.get_meta_props()
+        return base_model
diff --git a/nvflare/app_common/widgets/__init__.py b/nvflare/app_common/widgets/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/app_common/widgets/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/app_common/widgets/convert_to_fed_event.py b/nvflare/app_common/widgets/convert_to_fed_event.py
new file mode 100644
index 0000000000..fad836520b
--- /dev/null
+++ b/nvflare/app_common/widgets/convert_to_fed_event.py
@@ -0,0 +1,30 @@
+from typing import List
+
+from nvflare.apis.fl_constant import EventScope, FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.widgets.widget import Widget
+
+FED_EVENT_PREFIX = "fed."
+
+
+class ConvertToFedEvent(Widget):
+    def __init__(self, events_to_convert: List[str], fed_event_prefix=FED_EVENT_PREFIX):
+        Widget.__init__(self)
+        self.events_to_convert = events_to_convert
+        self.fed_event_prefix = fed_event_prefix
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type in self.events_to_convert:
+            event_scope = fl_ctx.get_prop(key=FLContextKey.EVENT_SCOPE, default=EventScope.LOCAL)
+            if event_scope == EventScope.FEDERATION:
+                # already a fed event
+                return
+            data = fl_ctx.get_prop(FLContextKey.EVENT_DATA, None)
+            if data is None:
+                self.log_error(fl_ctx, "Missing event data.")
+                return
+            if not isinstance(data, Shareable):
+                self.log_error(fl_ctx, f"Expect data to be shareable but got {type(data)}")
+                return
+            self.fire_fed_event(self.fed_event_prefix + event_type, data, fl_ctx)
diff --git a/nvflare/app_common/widgets/streaming.py b/nvflare/app_common/widgets/streaming.py
new file mode 100644
index 0000000000..999d08dc3b
--- /dev/null
+++ b/nvflare/app_common/widgets/streaming.py
@@ -0,0 +1,83 @@
+from abc import ABC, abstractmethod
+
+from nvflare.apis.analytix import Data as AnalyticsData
+from nvflare.apis.analytix import DataType as AnalyticsDataType
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FedEventHeader, FLContextKey, ReservedKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.widgets.widget import Widget
+
+EVENT = "log_analytics"
+
+
+class AnalyticsSender(Widget):
+    def __init__(self, event=EVENT):
+        super().__init__()
+        self.event = event
+
+    def write_scalar(self, fl_ctx, tag, scalar, **kwargs):
+        self.write(fl_ctx, tag, scalar, data_type=AnalyticsDataType.SCALAR, kwargs=kwargs)
+
+    def write_scalars(self, fl_ctx, tag, tag_scalar_dict, **kwargs):
+        self.write(fl_ctx, tag, tag_scalar_dict, data_type=AnalyticsDataType.SCALARS, kwargs=kwargs)
+
+    def write_image(self, fl_ctx, tag, image, **kwargs):
+        self.write(fl_ctx, tag, image, data_type=AnalyticsDataType.IMAGE, kwargs=kwargs)
+
+    def write_text(self, fl_ctx, tag, text, **kwargs):
+        self.write(fl_ctx, tag, text, data_type=AnalyticsDataType.TEXT, kwargs=kwargs)
+
+    def write(self, fl_ctx: FLContext, tag, value, data_type: AnalyticsDataType, kwargs=None):
+        if not isinstance(fl_ctx, FLContext):
+            raise TypeError("expect fl_ctx to be FLContext, but got {}".format(type(fl_ctx)))
+        data = AnalyticsData(tag=tag, value=value, data_type=data_type, kwargs=kwargs)
+        dxo = data.to_dxo()
+        fl_ctx.set_prop(key=FLContextKey.EVENT_DATA, value=dxo.to_shareable(), private=True, sticky=False)
+        self.fire_event(event_type=self.event, fl_ctx=fl_ctx)
+
+
+class AnalyticsReceiver(Widget, ABC):
+    def __init__(self, events=None):
+        FLComponent.__init__(self)
+        if events is None:
+            events = [EVENT, f"fed.{EVENT}"]
+        self.events = events
+
+    @abstractmethod
+    def initialize(self, fl_ctx: FLContext):
+        pass
+
+    @abstractmethod
+    def save(self, fl_ctx: FLContext, shareable: Shareable, record_origin: str):
+        pass
+
+    @abstractmethod
+    def finalize(self, fl_ctx: FLContext):
+        pass
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.initialize(fl_ctx)
+        elif event_type in self.events:
+            data = fl_ctx.get_prop(FLContextKey.EVENT_DATA, None)
+            if data is None:
+                self.log_error(fl_ctx, "Missing event data.")
+                return
+            if not isinstance(data, Shareable):
+                self.log_error(fl_ctx, f"Expect shareable but get {type(data)}")
+                return
+            record_origin = fl_ctx.get_identity_name()
+
+            # if fed event use peer name to save
+            if data.get_header(FedEventHeader.ORIGIN) is not None:
+                peer_name = data.get_peer_prop(ReservedKey.IDENTITY_NAME, None)
+                record_origin = peer_name
+
+            if record_origin is None:
+                self.log_error(fl_ctx, "record_origin can't be None.")
+                return
+            self.save(shareable=data, fl_ctx=fl_ctx, record_origin=record_origin)
+        elif event_type == EventType.END_RUN:
+            self.finalize(fl_ctx)
diff --git a/nvflare/app_common/workflows/__init__.py b/nvflare/app_common/workflows/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/app_common/workflows/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/app_common/workflows/cross_site_model_eval.py b/nvflare/app_common/workflows/cross_site_model_eval.py
new file mode 100644
index 0000000000..e6c0c897c8
--- /dev/null
+++ b/nvflare/app_common/workflows/cross_site_model_eval.py
@@ -0,0 +1,526 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import time
+from typing import Union
+
+from nvflare.apis.client import Client
+from nvflare.apis.controller_spec import ClientTask, Task
+from nvflare.apis.dxo import DXO, from_bytes, from_shareable
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.impl.controller import Controller
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.apis.workspace import Workspace
+from nvflare.app_common.abstract.formatter import Formatter
+from nvflare.app_common.abstract.model_locator import ModelLocator
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.app_event_type import AppEventType
+from nvflare.widgets.info_collector import GroupInfoCollector, InfoCollector
+
+
+class CrossSiteModelEval(Controller):
+    def __init__(
+        self,
+        task_check_period=0.5,
+        cross_val_dir=AppConstants.CROSS_VAL_DIR,
+        submit_model_timeout=600,
+        validation_timeout: int = 6000,
+        model_locator_id="",
+        formatter_id="",
+        submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL,
+        validation_task_name=AppConstants.TASK_VALIDATION,
+        cleanup_models=False,
+        participating_clients=None,
+        wait_for_clients_timeout=300,
+    ):
+        """Cross Site Model Validation workflow
+
+        Args:
+            task_check_period (float, optional): How often to check for new tasks or tasks being finished.
+                Defaults to 0.5.
+            cross_val_dir (str, optional): Path to cross site validation directory relative to run directory.
+                Defaults to "cross_site_val".
+            submit_model_timeout (int, optional): Timeout of submit_model_task. Defaults to 600 secs.
+            validation_timeout (int, optional): Timeout for validate_model task. Defaults to 6000 secs.
+            model_locator_id (str, optional): ID for model_locator component. Defaults to "".
+            formatter_id (str, optional): ID for formatter component. Defaults to "".
+            submit_model_task_name (str, optional): Name of submit_model task. Defaults to "".
+            validation_task_name (str, optional): Name of validate_model task. Defaults to "validate".
+            cleanup_models (bool, optional): Whether or not models should be deleted after run. Defaults to False.
+            participating_clients (list, optional): List of participating client names. If not provided, defaults
+                to all clients connected at start of controller.
+            wait_for_clients_timeout (int, optional): Timeout for clients to appear. Defaults to 300 secs
+        """
+        super(CrossSiteModelEval, self).__init__(task_check_period=task_check_period)
+
+        if not isinstance(task_check_period, float):
+            raise TypeError("task_check_period must be float.")
+        if not isinstance(cross_val_dir, str):
+            raise TypeError("cross_val_dir must be a string.")
+        if not isinstance(submit_model_timeout, int):
+            raise TypeError("submit_model_timeout must be int.")
+        if not isinstance(validation_timeout, int):
+            raise TypeError("validation_timeout must be int.")
+        if not isinstance(model_locator_id, str):
+            raise TypeError("model_locator_id must be a string.")
+        if not isinstance(formatter_id, str):
+            raise TypeError("formatter_id must be a string.")
+        if not isinstance(submit_model_task_name, str):
+            raise TypeError("submit_model_task_name must be a string.")
+        if not isinstance(validation_task_name, str):
+            raise TypeError("validation_task_name must be a string.")
+        if not isinstance(cleanup_models, bool):
+            raise TypeError("cleanup_models must be bool.")
+
+        if submit_model_timeout < 0:
+            raise ValueError("submit_model_timeout must be greater than or equal to 0.")
+        if validation_timeout < 0:
+            raise ValueError("model_validate_timeout must be greater than or equal to 0.")
+        if wait_for_clients_timeout < 0:
+            raise ValueError("wait_for_clients_timeout must be greater than or equal to 0.")
+
+        self._cross_val_dir = cross_val_dir
+        self._model_locator_id = model_locator_id
+        self._formatter_id = formatter_id
+        self._submit_model_task_name = submit_model_task_name
+        self._validation_task_name = validation_task_name
+        self._submit_model_timeout = submit_model_timeout
+        self._validation_timeout = validation_timeout
+        self._wait_for_clients_timeout = wait_for_clients_timeout
+        self._cleanup_models = cleanup_models
+        self._participating_clients = participating_clients
+
+        self._val_results = {}
+        self._server_models = {}
+        self._client_models = {}
+
+        self._formatter = None
+        self._cross_val_models_dir = None
+        self._cross_val_results_dir = None
+        self._model_locator = None
+
+    def start_controller(self, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        if not engine:
+            self.system_panic("Engine not found. Workflow exiting.", fl_ctx)
+            return
+
+        # If the list of participating clients is not provided, include all clients currently available.
+        if not self._participating_clients:
+            clients = engine.get_clients()
+            self._participating_clients = [c.name for c in clients]
+        else:
+            assert isinstance(self._participating_clients, list), "participating_clients must be a list"
+            assert all(isinstance(x, str) for x in self._participating_clients), "participating_clients must be strings"
+
+        # Create shareable dirs for models and results
+        workspace: Workspace = engine.get_workspace()
+        run_dir = workspace.get_run_dir(fl_ctx.get_run_number())
+        cross_val_path = os.path.join(run_dir, self._cross_val_dir)
+        self._cross_val_models_dir = os.path.join(cross_val_path, AppConstants.CROSS_VAL_MODEL_DIR_NAME)
+        self._cross_val_results_dir = os.path.join(cross_val_path, AppConstants.CROSS_VAL_RESULTS_DIR_NAME)
+
+        # Fire the init event.
+        fl_ctx.set_prop(AppConstants.CROSS_VAL_MODEL_PATH, self._cross_val_models_dir)
+        fl_ctx.set_prop(AppConstants.CROSS_VAL_RESULTS_PATH, self._cross_val_results_dir)
+        self.fire_event(AppEventType.CROSS_VAL_INIT, fl_ctx)
+
+        # Cleanup/create the cross val models and results directories
+        if os.path.exists(self._cross_val_models_dir):
+            shutil.rmtree(self._cross_val_models_dir)
+        if os.path.exists(self._cross_val_results_dir):
+            shutil.rmtree(self._cross_val_results_dir)
+
+        # Recreate new directories.
+        os.makedirs(self._cross_val_models_dir)
+        os.makedirs(self._cross_val_results_dir)
+
+        # Get components
+        if self._model_locator_id:
+            self._model_locator = engine.get_component(self._model_locator_id)
+            if not isinstance(self._model_locator, ModelLocator):
+                self.system_panic(
+                    reason="bad model locator {}: expect ModelLocator but got {}".format(
+                        self._model_locator_id, type(self._model_locator)
+                    ),
+                    fl_ctx=fl_ctx,
+                )
+                return
+
+        if self._formatter_id:
+            self._formatter = engine.get_component(self._formatter_id)
+            if not isinstance(self._formatter, Formatter):
+                self.system_panic(
+                    reason=f"formatter {self._formatter_id} is not an instance of Formatter.", fl_ctx=fl_ctx
+                )
+                return
+
+        if not self._formatter:
+            self.log_info(fl_ctx, "Formatter not found. Stats will not be printed.")
+
+        for c_name in self._participating_clients:
+            self._client_models[c_name] = None
+            self._val_results[c_name] = {}
+
+    def control_flow(self, abort_signal: Signal, fl_ctx: FLContext):
+        try:
+            # wait until there are some clients
+            engine = fl_ctx.get_engine()
+            start_time = time.time()
+            while not self._participating_clients:
+                self._participating_clients = engine.get_clients()
+                if time.time() - start_time > self._wait_for_clients_timeout:
+                    self.log_info(fl_ctx, "No clients available - quit model validation.")
+                    return
+
+                self.log_info(fl_ctx, "No clients available - waiting ...")
+                time.sleep(2.0)
+                if abort_signal.triggered:
+                    self.log_info(fl_ctx, "Abort signal triggered. Finishing model validation.")
+                    return
+
+            self.log_info(fl_ctx, f"Beginning model validation with clients: {self._participating_clients}.")
+
+            if self._submit_model_task_name:
+                submit_model_task = Task(
+                    name=self._submit_model_task_name,
+                    data=Shareable(),
+                    result_received_cb=self._receive_local_model_cb,
+                    timeout=self._submit_model_timeout,
+                )
+                self.broadcast(
+                    task=submit_model_task,
+                    targets=self._participating_clients,
+                    fl_ctx=fl_ctx,
+                    min_responses=len(self._participating_clients),
+                )
+
+            if abort_signal.triggered:
+                self.log_info(fl_ctx, "Abort signal triggered. Finishing model validation.")
+                return
+
+            # Load server models and assign those tasks
+            if self._model_locator:
+                success = self._locate_server_models(fl_ctx)
+                if not success:
+                    return
+
+                for server_model in self._server_models:
+                    self._send_validation_task(server_model, fl_ctx)
+            else:
+                self.log_info(fl_ctx, "ModelLocator not present. No server models will be included.")
+
+            while self.get_num_standing_tasks():
+                if abort_signal.triggered:
+                    self.log_info(fl_ctx, "Abort signal triggered. Finishing cross site validation.")
+                    return
+                self.log_debug(fl_ctx, "Checking standing tasks to see if cross site validation finished.")
+                time.sleep(self._task_check_period)
+        except BaseException as e:
+            error_msg = f"Exception in cross site validator control_flow: {e.__str__()}"
+            self.log_exception(fl_ctx, error_msg)
+            self.system_panic(error_msg, fl_ctx)
+
+    def stop_controller(self, fl_ctx: FLContext):
+        self.cancel_all_tasks(fl_ctx=fl_ctx)
+
+        if self._cleanup_models:
+            self.log_info(fl_ctx, "Removing local models kept for validation.")
+            for model_name, model_path in self._server_models.items():
+                if model_path and os.path.isfile(model_path):
+                    os.remove(model_path)
+                    self.log_debug(fl_ctx, f"Removing server model {model_name} at {model_path}.")
+            for model_name, model_path in self._client_models.items():
+                if model_path and os.path.isfile(model_path):
+                    os.remove(model_path)
+                    self.log_debug(fl_ctx, f"Removing client {model_name}'s model at {model_path}.")
+
+    def _receive_local_model_cb(self, client_task: ClientTask, fl_ctx: FLContext):
+        client_name = client_task.client.name
+        result: Shareable = client_task.result
+
+        self._accept_local_model(client_name=client_name, result=result, fl_ctx=fl_ctx)
+
+        # Cleanup task result
+        client_task.result = None
+
+    def _before_send_validate_task_cb(self, client_task: ClientTask, fl_ctx: FLContext):
+        model_name = client_task.task.props[AppConstants.MODEL_OWNER]
+
+        try:
+            model_dxo: DXO = self._load_validation_content(model_name, self._cross_val_models_dir, fl_ctx)
+        except ValueError as v_e:
+            reason = f"Error in loading model shareable for {model_name}. CrossSiteValidator exiting."
+            self.log_error(fl_ctx, reason)
+            self.system_panic(reason, fl_ctx)
+            return
+
+        if not model_dxo:
+            self.system_panic(
+                f"Model contents for {model_name} not found in {self._cross_val_models_dir}. "
+                "CrossSiteValidator exiting",
+                fl_ctx=fl_ctx,
+            )
+            return
+
+        model_shareable = model_dxo.to_shareable()
+        model_shareable.set_header(AppConstants.MODEL_OWNER, model_name)
+        model_shareable.add_cookie(AppConstants.MODEL_OWNER, model_name)
+        client_task.task.data = model_shareable
+
+        fl_ctx.set_prop(AppConstants.DATA_CLIENT, client_task.client, private=True, sticky=False)
+        fl_ctx.set_prop(AppConstants.MODEL_OWNER, model_name, private=True, sticky=False)
+        fl_ctx.set_prop(AppConstants.MODEL_TO_VALIDATE, model_shareable, private=True, sticky=False)
+        fl_ctx.set_prop(AppConstants.PARTICIPATING_CLIENTS, self._participating_clients, private=True, sticky=False)
+        self.fire_event(AppEventType.SEND_MODEL_FOR_VALIDATION, fl_ctx)
+
+    def _after_send_validate_task_cb(self, client_task: ClientTask, fl_ctx: FLContext):
+        # Once task is sent clear data to restore memory
+        client_task.task.data = None
+
+    def _receive_val_result_cb(self, client_task: ClientTask, fl_ctx: FLContext):
+        # Find name of the client sending this
+        result = client_task.result
+        client_name = client_task.client.name
+
+        self._accept_val_result(client_name=client_name, result=result, fl_ctx=fl_ctx)
+
+        client_task.result = None
+
+    def _locate_server_models(self, fl_ctx: FLContext) -> bool:
+        # Load models from model_locator
+        self.log_info(fl_ctx, "Locating server models.")
+        server_model_names = self._model_locator.get_model_names(fl_ctx)
+
+        unique_names = []
+        for name in server_model_names:
+            # Get the model
+            dxo = self._model_locator.locate_model(name, fl_ctx)
+            if not isinstance(dxo, DXO):
+                self.system_panic(f"ModelLocator produced invalid data: expect DXO but got {type(dxo)}.", fl_ctx)
+                return False
+
+            # Save to workspace
+            unique_name = "SRV_" + name
+            unique_names.append(unique_name)
+            try:
+                save_path = self._save_validation_content(unique_name, self._cross_val_models_dir, dxo, fl_ctx)
+            except:
+                self.log_exception(fl_ctx, f"Unable to save shareable contents of server model {unique_name}")
+                self.system_panic(f"Unable to save shareable contents of server model {unique_name}", fl_ctx)
+                return False
+
+            self._server_models[unique_name] = save_path
+            self._val_results[unique_name] = {}
+
+        if unique_names:
+            self.log_info(fl_ctx, f"Server models loaded: {unique_names}.")
+        else:
+            self.log_info(fl_ctx, "no server models to validate!")
+        return True
+
+    def _accept_local_model(self, client_name: str, result: Shareable, fl_ctx: FLContext):
+        fl_ctx.set_prop(AppConstants.RECEIVED_MODEL, result, private=False, sticky=False)
+        fl_ctx.set_prop(AppConstants.RECEIVED_MODEL_OWNER, client_name, private=False, sticky=False)
+        fl_ctx.set_prop(AppConstants.CROSS_VAL_DIR, self._cross_val_dir, private=False, sticky=False)
+        self.fire_event(AppEventType.RECEIVE_BEST_MODEL, fl_ctx)
+
+        # get return code
+        rc = result.get_return_code()
+        if rc and rc != ReturnCode.OK:
+            # Raise errors if bad peer context or execution exception.
+            if rc in [ReturnCode.MISSING_PEER_CONTEXT, ReturnCode.BAD_PEER_CONTEXT]:
+                self.log_error(fl_ctx, "Peer context is bad or missing. No model submitted for this client.")
+            elif rc in [ReturnCode.EXECUTION_EXCEPTION, ReturnCode.TASK_UNKNOWN]:
+                self.log_error(
+                    fl_ctx, "Execution Exception on client during model submission. No model submitted for this client."
+                )
+            # Ignore contribution if result invalid.
+            elif rc in [
+                ReturnCode.EXECUTION_RESULT_ERROR,
+                ReturnCode.TASK_DATA_FILTER_ERROR,
+                ReturnCode.TASK_RESULT_FILTER_ERROR,
+                ReturnCode.TASK_UNKNOWN,
+            ]:
+                self.log_error(fl_ctx, "Execution result is not a shareable. Model submission will be ignored.")
+            else:
+                self.log_error(fl_ctx, "Return code set. Model submission from client will be ignored.")
+        else:
+            # Save shareable in models directory.
+            try:
+                self.log_debug(fl_ctx, "Extracting DXO from shareable.")
+                dxo = from_shareable(result)
+                save_path = self._save_validation_content(client_name, self._cross_val_models_dir, dxo, fl_ctx)
+            except ValueError as v_e:
+                self.log_error(
+                    fl_ctx, f"Unable to save shareable contents of {client_name}'s model. Exception: {str(v_e)}"
+                )
+                self.log_warning(fl_ctx, f"Ignoring client {client_name}'s model.")
+                return
+
+            self.log_info(fl_ctx, f"Received local model from client {client_name}.")
+
+            self._client_models[client_name] = save_path
+
+            # Send a model to this client to validate
+            self._send_validation_task(client_name, fl_ctx)
+
+    def _send_validation_task(self, model_name: str, fl_ctx: FLContext):
+        self.log_info(fl_ctx, f"Sending {model_name} model to all participating clients for validation.")
+
+        # Create validation task and broadcast to all participating clients.
+        task = Task(
+            name=self._validation_task_name,
+            data=Shareable(),
+            before_task_sent_cb=self._before_send_validate_task_cb,
+            after_task_sent_cb=self._after_send_validate_task_cb,
+            result_received_cb=self._receive_val_result_cb,
+            timeout=self._validation_timeout,
+            props={AppConstants.MODEL_OWNER: model_name},
+        )
+
+        self.broadcast(
+            task=task,
+            fl_ctx=fl_ctx,
+            targets=self._participating_clients,
+            min_responses=len(self._participating_clients),
+            wait_time_after_min_received=0,
+        )
+
+    def _accept_val_result(self, client_name: str, result: Shareable, fl_ctx: FLContext):
+        model_owner = result.get_cookie(AppConstants.MODEL_OWNER, "")
+
+        # Fire event. This needs to be a new local context per each client
+        fl_ctx.set_prop(AppConstants.MODEL_OWNER, model_owner, private=True, sticky=False)
+        fl_ctx.set_prop(AppConstants.DATA_CLIENT, client_name, private=True, sticky=False)
+        fl_ctx.set_prop(AppConstants.VALIDATION_RESULT, result, private=True, sticky=False)
+        self.fire_event(AppEventType.VALIDATION_RESULT_RECEIVED, fl_ctx)
+
+        rc = result.get_return_code()
+        if rc and rc != ReturnCode.OK:
+            # Raise errors if bad peer context or execution exception.
+            if rc in [ReturnCode.MISSING_PEER_CONTEXT, ReturnCode.BAD_PEER_CONTEXT]:
+                self.log_error(fl_ctx, "Peer context is bad or missing.")
+            elif rc in [ReturnCode.EXECUTION_EXCEPTION, ReturnCode.TASK_UNKNOWN]:
+                self.log_error(fl_ctx, "Execution Exception in model validation.")
+            elif rc in [
+                ReturnCode.EXECUTION_RESULT_ERROR,
+                ReturnCode.TASK_DATA_FILTER_ERROR,
+                ReturnCode.TASK_RESULT_FILTER_ERROR,
+            ]:
+                self.log_error(fl_ctx, "Execution result is not a shareable. Validation results will be ignored.")
+            else:
+                self.log_error(
+                    fl_ctx,
+                    f"Client {client_name} sent results for validating {model_owner} model with return code set."
+                    " Logging empty results.",
+                )
+
+            self._val_results[client_name][model_owner] = {}
+        else:
+            save_file_name = client_name + "_" + model_owner
+
+            try:
+                dxo = from_shareable(result)
+                self._save_validation_content(save_file_name, self._cross_val_results_dir, dxo, fl_ctx)
+                self._val_results[client_name][model_owner] = os.path.join(self._cross_val_results_dir, save_file_name)
+
+                self.log_info(fl_ctx, f"Client {client_name} sent results for validating {model_owner} model.")
+            except ValueError as v_e:
+                reason = (
+                    f"Unable to save validation result from {client_name} of {model_owner}'s model. "
+                    f"Exception: {str(v_e)}"
+                )
+                self.log_exception(fl_ctx, reason)
+
+    def _save_validation_content(self, name: str, save_dir: str, dxo: DXO, fl_ctx: FLContext) -> str:
+        """Saves shareable to given directory within the app_dir
+
+        Args:
+            name (str): Name of shareable
+            save_dir (str): Relative path to directory in which to save
+            shareable (Shareable): Shareable object
+            fl_ctx (FLContext): FLContext object
+
+        Returns:
+            str: Path to the file saved.
+        """
+        # Save the model with name as the filename to shareable directory
+        data_filename = os.path.join(save_dir, name)
+
+        try:
+            bytes_to_save = dxo.to_bytes()
+        except Exception as e:
+            raise ValueError(f"Unable to extract shareable contents. Exception: {(e.__str__())}") from e
+
+        # Save contents to path
+        try:
+            with open(data_filename, "wb") as f:
+                f.write(bytes_to_save)
+        except Exception as e:
+            raise ValueError(f"Unable to save shareable contents: {str(e)}") from e
+
+        self.log_debug(fl_ctx, f"Saved cross validation model with name: {name}.")
+
+        return data_filename
+
+    def _load_validation_content(self, name: str, load_dir: str, fl_ctx: FLContext) -> Union[DXO, None]:
+        # Load shareable from disk
+        shareable_filename = os.path.join(load_dir, name)
+        dxo: DXO = None
+
+        # load shareable
+        try:
+            with open(shareable_filename, "rb") as f:
+                data = f.read()
+
+            dxo: DXO = from_bytes(data)
+
+            self.log_debug(fl_ctx, f"Loading cross validation shareable content with name: {name}.")
+        except Exception as e:
+            raise ValueError(f"Exception in loading shareable content for {name}: {str(e)}")
+
+        return dxo
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        super().handle_event(event_type=event_type, fl_ctx=fl_ctx)
+        if event_type == InfoCollector.EVENT_TYPE_GET_STATS:
+            if self._formatter:
+                collector = fl_ctx.get_prop(InfoCollector.CTX_KEY_STATS_COLLECTOR, None)
+                if collector:
+                    assert isinstance(collector, GroupInfoCollector)
+
+                    fl_ctx.set_prop(AppConstants.VALIDATION_RESULT, self._val_results, private=True, sticky=False)
+                    val_info = self._formatter.format(fl_ctx)
+
+                    collector.add_info(
+                        group_name=self._name,
+                        info={"val_results": val_info},
+                    )
+            else:
+                self.log_warning(fl_ctx, "No formatter provided. Validation results can't be printed.")
+
+    def process_result_of_unknown_task(
+        self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext
+    ):
+        if task_name == self._submit_model_task_name:
+            self._accept_local_model(client_name=client.name, result=result, fl_ctx=fl_ctx)
+        elif task_name == self._validation_task_name:
+            self._accept_val_result(client_name=client.name, result=result, fl_ctx=fl_ctx)
+        else:
+            self.log_error(fl_ctx, "Ignoring result from unknown task.")
diff --git a/nvflare/app_common/workflows/cyclic_ctl.py b/nvflare/app_common/workflows/cyclic_ctl.py
new file mode 100644
index 0000000000..d922d99fe3
--- /dev/null
+++ b/nvflare/app_common/workflows/cyclic_ctl.py
@@ -0,0 +1,141 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+
+from nvflare.apis.client import Client
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.impl.controller import ClientTask, Controller, Task
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.learnable_persistor import LearnablePersistor
+from nvflare.app_common.abstract.shareable_generator import ShareableGenerator
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.app_event_type import AppEventType
+
+
+class CyclicController(Controller):
+    """
+    A sample implementation to demonstrate how to use relay method for Cyclic Federated Learning
+    """
+
+    def __init__(
+        self,
+        num_rounds: int = 5,
+        task_assignment_timeout: int = 10,
+        persistor_id="persistor",
+        shareable_generator_id="shareable_generator",
+        task_name="train",
+    ):
+        super().__init__()
+        if not isinstance(num_rounds, int):
+            raise TypeError("num_rounds must be int.")
+        if not isinstance(task_assignment_timeout, int):
+            raise TypeError("task_assignment_timeout must be int.")
+        if not isinstance(persistor_id, str):
+            raise TypeError("persistor_id must be a string.")
+        if not isinstance(shareable_generator_id, str):
+            raise TypeError("shareable_generator_id must be a string.")
+        if not isinstance(task_name, str):
+            raise TypeError("task_name must be a string.")
+        self.num_rounds = num_rounds
+        self.persistor_id = persistor_id
+        self.shareable_generator_id = shareable_generator_id
+        self.task_assignment_timeout = task_assignment_timeout
+        self.task_name = task_name
+
+    def start_controller(self, fl_ctx: FLContext):
+        self.log_debug(fl_ctx, "starting controller")
+        self.persistor = fl_ctx.get_engine().get_component(self.persistor_id)
+        self.shareable_generator = fl_ctx.get_engine().get_component(self.shareable_generator_id)
+        if not isinstance(self.persistor, LearnablePersistor):
+            self.system_panic(f"Persistor {self.persistor_id} must be a Persistor instance", fl_ctx)
+        if not isinstance(self.shareable_generator, ShareableGenerator):
+            self.system_panic(
+                f"Shareable generator {self.shareable_generator_id} must be a Shareable Generator instance", fl_ctx
+            )
+        self.last_learnable = self.persistor.load(fl_ctx)
+        fl_ctx.set_prop(AppConstants.GLOBAL_MODEL, self.last_learnable, private=True, sticky=True)
+        self.fire_event(AppEventType.INITIAL_MODEL_LOADED, fl_ctx)
+
+    def _process_result(self, client_task: ClientTask, fl_ctx: FLContext):
+        # submitted shareable is stored in client_task.result
+        # we need to update task.data with that shareable so the next target
+        # will get the updated shareable
+        task = client_task.task
+
+        # update the global learnable with the received result (shareable)
+        # e.g. the received result could be weight_diffs, the learnable could be full weights.
+        self.last_learnable = self.shareable_generator.shareable_to_learnable(client_task.result, fl_ctx)
+
+        # prepare task shareable data for next client
+        task.data = self.shareable_generator.learnable_to_shareable(self.last_learnable, fl_ctx)
+
+    def control_flow(self, abort_signal: Signal, fl_ctx: FLContext):
+        try:
+            engine = fl_ctx.get_engine()
+            self.log_debug(fl_ctx, "Cyclic starting.")
+
+            for current_round in range(self.num_rounds):
+                if abort_signal.triggered:
+                    return
+
+                self.log_debug(fl_ctx, f"Starting {current_round=}.")
+                fl_ctx.set_prop(AppConstants.CURRENT_ROUND, current_round, private=True, sticky=False)
+
+                # Task for one cyclic
+                targets = engine.get_clients()
+                random.shuffle(targets)
+                targets_names = [t.name for t in targets]
+                self.log_debug(fl_ctx, f"Relay on {targets_names}")
+
+                shareable = self.shareable_generator.learnable_to_shareable(self.last_learnable, fl_ctx)
+                shareable.set_header(AppConstants.CURRENT_ROUND, current_round)
+
+                task = Task(
+                    name=self.task_name,
+                    data=shareable,
+                    result_received_cb=self._process_result,
+                )
+
+                self.relay_and_wait(
+                    task=task,
+                    targets=targets,
+                    task_assignment_timeout=self.task_assignment_timeout,
+                    fl_ctx=fl_ctx,
+                    dynamic_targets=False,
+                    abort_signal=abort_signal,
+                )
+                self.persistor.save(self.last_learnable, fl_ctx)
+                self.log_debug(fl_ctx, f"Ending {current_round=}.")
+
+            self.log_debug(fl_ctx, "Cyclic ended.")
+        except BaseException as e:
+            error_msg = f"Cyclic control_flow exception {e}"
+            self.log_error(fl_ctx, error_msg)
+            self.system_panic(str(e), fl_ctx)
+
+    def stop_controller(self, fl_ctx: FLContext):
+        self.persistor.save(learnable=self.last_learnable, fl_ctx=fl_ctx)
+        self.log_debug(fl_ctx, "controller stopped")
+
+    def process_result_of_unknown_task(
+        self,
+        client: Client,
+        task_name: str,
+        client_task_id: str,
+        result: Shareable,
+        fl_ctx: FLContext,
+    ):
+        self.log_warning(fl_ctx, f"Dropped result of unknown task: {task_name} from client {client.name}.")
diff --git a/nvflare/app_common/workflows/global_model_eval.py b/nvflare/app_common/workflows/global_model_eval.py
new file mode 100644
index 0000000000..fd7423ffb0
--- /dev/null
+++ b/nvflare/app_common/workflows/global_model_eval.py
@@ -0,0 +1,64 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.app_common.app_constant import AppConstants
+
+from .cross_site_model_eval import CrossSiteModelEval
+
+
+class GlobalModelEval(CrossSiteModelEval):
+    def __init__(
+        self,
+        task_check_period=0.5,
+        cross_val_dir=AppConstants.CROSS_VAL_DIR,
+        validation_timeout: int = 6000,
+        model_locator_id="",
+        formatter_id="",
+        validation_task_name=AppConstants.TASK_VALIDATION,
+        cleanup_models=False,
+        participating_clients=None,
+        wait_for_clients_timeout=300,
+    ):
+        """Cross Site Model Validation workflow
+
+        Args:
+            task_check_period (float, optional): How often to check for new tasks or tasks being finished.
+                Defaults to 0.5.
+            cross_val_dir (str, optional): Path to cross site validation directory relative to run directory.
+                Defaults to "cross_site_val".
+            validation_timeout (int, optional): Timeout for validate_model task. Defaults to 6000.
+            model_locator_id (str, optional): ID for model_locator component. Defaults to None.
+            formatter_id (str, optional): ID for formatter component. Defaults to None.
+            validation_task_name (str, optional): Name of validate_model task. Defaults to "validate".
+            cleanup_models (bool, optional): Whether or not models should be deleted after run. Defaults to False.
+            participating_clients (list, optional): List of participating client names. If not provided, defaults
+                to all clients connected at start of controller.
+            wait_for_clients_timeout (int, optional): Timeout for clients to appear. Defaults to 300 secs
+        """
+        if not model_locator_id:
+            raise ValueError("missing required model_locator_id")
+
+        CrossSiteModelEval.__init__(
+            self,
+            task_check_period=task_check_period,
+            cross_val_dir=cross_val_dir,
+            validation_timeout=validation_timeout,
+            model_locator_id=model_locator_id,
+            formatter_id=formatter_id,
+            validation_task_name=validation_task_name,
+            submit_model_task_name="",
+            cleanup_models=cleanup_models,
+            participating_clients=participating_clients,
+            wait_for_clients_timeout=wait_for_clients_timeout,
+        )
diff --git a/nvflare/app_common/workflows/scatter_and_gather.py b/nvflare/app_common/workflows/scatter_and_gather.py
new file mode 100644
index 0000000000..3aea41ad46
--- /dev/null
+++ b/nvflare/app_common/workflows/scatter_and_gather.py
@@ -0,0 +1,310 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import traceback
+
+from nvflare.apis.client import Client
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.impl.controller import ClientTask, Controller, Task
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.aggregator import Aggregator
+from nvflare.app_common.abstract.learnable_persistor import LearnablePersistor
+from nvflare.app_common.abstract.shareable_generator import ShareableGenerator
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.app_event_type import AppEventType
+from nvflare.widgets.info_collector import GroupInfoCollector, InfoCollector
+
+
+class ScatterAndGather(Controller):
+    def __init__(
+        self,
+        min_clients: int = 1,
+        num_rounds: int = 5,
+        start_round: int = 0,
+        wait_time_after_min_received: int = 10,
+        aggregator_id=AppConstants.DEFAULT_AGGREGATOR_ID,
+        persistor_id=AppConstants.DEFAULT_PERSISTOR_ID,
+        shareable_generator_id=AppConstants.DEFAULT_SHAREABLE_GENERATOR_ID,
+        train_task_name=AppConstants.TASK_TRAIN,
+        train_timeout: int = 0,
+        ignore_result_error: bool = False,
+    ):
+        """FederatedAveraging Workflow. The ScatterAndGather workflow defines Federated training on all clients.
+        The model persistor (persistor_id) is used to load the initial global model which is sent to all clients.
+        Each clients sends it's updated weights after local training which is aggregated (aggregator_id). The
+        shareable generator is used to convert the aggregated weights to shareable and shareable back to weights.
+        The model_persistor also saves the model after training.
+
+        Args:
+            min_clients (int, optional): Min number of clients in training. Defaults to 1.
+            num_rounds (int, optional): The total number of training rounds. Defaults to 5.
+            start_round (int, optional): Start round for training. Defaults to 0.
+            wait_time_after_min_received (int, optional): Time to wait before beginning aggregation after
+                contributions received. Defaults to 10.
+            train_timeout (int, optional): Time to wait for clients to do local training.
+            aggregator_id (str, optional): ID of the aggregator component. Defaults to "aggregator".
+            persistor_id (str, optional): ID of the persistor component. Defaults to "persistor".
+            shareable_generator_id (str, optional): ID of the shareable generator. Defaults to "shareable_generator".
+            train_task_name (str, optional): Name of the train task. Defaults to "train".
+        """
+        Controller.__init__(self)
+
+        # Check arguments
+        if not isinstance(min_clients, int):
+            raise TypeError("min_clients must be int.")
+        if not isinstance(num_rounds, int):
+            raise TypeError("num_rounds must be int.")
+        if not isinstance(start_round, int):
+            raise TypeError("start_round must be int.")
+        if not isinstance(wait_time_after_min_received, int):
+            raise TypeError("wait_time_after_min_received must be int.")
+        if not isinstance(train_timeout, int):
+            raise TypeError("train_timeout must be int.")
+        if not isinstance(aggregator_id, str):
+            raise TypeError("aggregator_id must be a string.")
+        if not isinstance(persistor_id, str):
+            raise TypeError("persistor_id must be a string.")
+        if not isinstance(shareable_generator_id, str):
+            raise TypeError("shareable_generator_id must be a string.")
+        if not isinstance(train_task_name, str):
+            raise TypeError("train_task_name must be a string.")
+        if min_clients <= 0:
+            raise ValueError("min_clients must be greater than 0.")
+        if num_rounds < 0:
+            raise ValueError("num_rounds must be greater than or equal to 0.")
+        if start_round < 0:
+            raise ValueError("start_round must be greater than or equal to 0.")
+        if wait_time_after_min_received < 0:
+            raise ValueError("wait_time_after_min_received must be greater than or equal to 0.")
+
+        self.aggregator_id = aggregator_id
+        self.persistor_id = persistor_id
+        self.shareable_generator_id = shareable_generator_id
+        self.train_task_name = train_task_name
+        self.aggregator = None
+        self.persistor = None
+        self.shareable_gen = None
+
+        # config data
+        self._min_clients = min_clients
+        self._num_rounds = num_rounds
+        self._wait_time_after_min_received = wait_time_after_min_received  # 5 minutes
+        self._start_round = start_round
+        self._train_timeout = train_timeout
+        self.ignore_result_error = ignore_result_error
+
+        # workflow phases: init, train, validate
+        self._phase = AppConstants.PHASE_INIT
+        self._global_weights = None
+        self._current_round = None
+
+    def start_controller(self, fl_ctx: FLContext) -> None:
+        self.log_info(fl_ctx, "Initializing ScatterAndGather workflow.")
+        self._phase = AppConstants.PHASE_INIT
+        engine = fl_ctx.get_engine()
+        if not engine:
+            self.system_panic("Engine not found. ScatterAndGather exiting.", fl_ctx)
+            return
+
+        self.aggregator = engine.get_component(self.aggregator_id)
+        if not isinstance(self.aggregator, Aggregator):
+            self.system_panic(f"aggregator {self.aggregator_id} must be an Aggregator type object.", fl_ctx)
+            return
+
+        self.shareable_gen = engine.get_component(self.shareable_generator_id)
+        if not isinstance(self.shareable_gen, ShareableGenerator):
+            self.system_panic(
+                f"Shareable generator {self.shareable_generator_id} must be a ShareableGenerator type object.", fl_ctx
+            )
+            return
+
+        self.persistor = engine.get_component(self.persistor_id)
+        if not isinstance(self.persistor, LearnablePersistor):
+            self.system_panic(
+                f"Model Persistor {self.persistor_id} must be a LearnablePersistor type object.",
+                fl_ctx,
+            )
+            return
+
+        # initialize global model
+        fl_ctx.set_prop(AppConstants.START_ROUND, self._start_round, private=True, sticky=True)
+        fl_ctx.set_prop(AppConstants.NUM_ROUNDS, self._num_rounds, private=True, sticky=False)
+        self._global_weights = self.persistor.load(fl_ctx)
+        fl_ctx.set_prop(AppConstants.GLOBAL_MODEL, self._global_weights, private=True, sticky=True)
+        self.fire_event(AppEventType.INITIAL_MODEL_LOADED, fl_ctx)
+
+    def control_flow(self, abort_signal: Signal, fl_ctx: FLContext) -> None:
+        try:
+
+            self.log_info(fl_ctx, "Beginning ScatterAndGather training phase.")
+            self._phase = AppConstants.PHASE_TRAIN
+
+            fl_ctx.set_prop(AppConstants.PHASE, self._phase, private=True, sticky=False)
+            fl_ctx.set_prop(AppConstants.NUM_ROUNDS, self._num_rounds, private=True, sticky=False)
+            self.fire_event(AppEventType.TRAINING_STARTED, fl_ctx)
+
+            for self._current_round in range(self._start_round, self._start_round + self._num_rounds):
+                if self._check_abort_signal(fl_ctx, abort_signal):
+                    return
+
+                self.log_info(fl_ctx, f"Round {self._current_round} started.")
+                fl_ctx.set_prop(AppConstants.GLOBAL_MODEL, self._global_weights, private=True, sticky=True)
+                fl_ctx.set_prop(AppConstants.CURRENT_ROUND, self._current_round, private=True, sticky=False)
+                self.fire_event(AppEventType.ROUND_STARTED, fl_ctx)
+
+                # Create train_task
+                data_shareable: Shareable = self.shareable_gen.learnable_to_shareable(self._global_weights, fl_ctx)
+                data_shareable.set_header(AppConstants.CURRENT_ROUND, self._current_round)
+                data_shareable.set_header(AppConstants.NUM_ROUNDS, self._num_rounds)
+                data_shareable.add_cookie(AppConstants.CONTRIBUTION_ROUND, self._current_round)
+
+                train_task = Task(
+                    name=self.train_task_name,
+                    data=data_shareable,
+                    props={},
+                    timeout=self._train_timeout,
+                    before_task_sent_cb=self._prepare_train_task_data,
+                    result_received_cb=self._process_train_result,
+                )
+
+                self.broadcast_and_wait(
+                    task=train_task,
+                    min_responses=self._min_clients,
+                    wait_time_after_min_received=self._wait_time_after_min_received,
+                    fl_ctx=fl_ctx,
+                    abort_signal=abort_signal,
+                )
+
+                if self._check_abort_signal(fl_ctx, abort_signal):
+                    return
+
+                self.fire_event(AppEventType.BEFORE_AGGREGATION, fl_ctx)
+                aggr_result = self.aggregator.aggregate(fl_ctx)
+                fl_ctx.set_prop(AppConstants.AGGREGATION_RESULT, aggr_result, private=True, sticky=False)
+                self.fire_event(AppEventType.AFTER_AGGREGATION, fl_ctx)
+
+                if self._check_abort_signal(fl_ctx, abort_signal):
+                    return
+
+                self.fire_event(AppEventType.BEFORE_SHAREABLE_TO_LEARNABLE, fl_ctx)
+                self._global_weights = self.shareable_gen.shareable_to_learnable(aggr_result, fl_ctx)
+                fl_ctx.set_prop(AppConstants.GLOBAL_MODEL, self._global_weights, private=True, sticky=True)
+                fl_ctx.sync_sticky()
+                self.fire_event(AppEventType.AFTER_SHAREABLE_TO_LEARNABLE, fl_ctx)
+
+                if self._check_abort_signal(fl_ctx, abort_signal):
+                    return
+
+                self.fire_event(AppEventType.BEFORE_LEARNABLE_PERSIST, fl_ctx)
+                self.persistor.save(self._global_weights, fl_ctx)
+                self.fire_event(AppEventType.AFTER_LEARNABLE_PERSIST, fl_ctx)
+
+                self.fire_event(AppEventType.ROUND_DONE, fl_ctx)
+                self.log_info(fl_ctx, f"Round {self._current_round} finished.")
+
+            self._phase = AppConstants.PHASE_FINISHED
+            self.log_info(fl_ctx, "Finished ScatterAndGather Training.")
+        except BaseException as e:
+            traceback.print_exc()
+            error_msg = f"Exception in ScatterAndGather control_flow: {e}"
+            self.log_exception(fl_ctx, error_msg)
+            self.system_panic(str(e), fl_ctx)
+
+    def stop_controller(self, fl_ctx: FLContext) -> None:
+        self._phase = AppConstants.PHASE_FINISHED
+        self.cancel_all_tasks()
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        super().handle_event(event_type, fl_ctx)
+        if event_type == InfoCollector.EVENT_TYPE_GET_STATS:
+            collector = fl_ctx.get_prop(InfoCollector.CTX_KEY_STATS_COLLECTOR, None)
+            if collector:
+                assert isinstance(collector, GroupInfoCollector)
+
+                collector.add_info(
+                    group_name=self._name,
+                    info={"phase": self._phase, "current_round": self._current_round, "num_rounds": self._num_rounds},
+                )
+
+    def _prepare_train_task_data(self, client_task: ClientTask, fl_ctx: FLContext) -> None:
+        fl_ctx.set_prop(AppConstants.TRAIN_SHAREABLE, client_task.task.data, private=True, sticky=False)
+        self.fire_event(AppEventType.BEFORE_TRAIN_TASK, fl_ctx)
+
+    def _process_train_result(self, client_task: ClientTask, fl_ctx: FLContext) -> None:
+        result = client_task.result
+        client_name = client_task.client.name
+
+        self._accept_train_result(client_name=client_name, result=result, fl_ctx=fl_ctx)
+
+        # Cleanup task result
+        client_task.result = None
+
+    def process_result_of_unknown_task(
+        self, client: Client, task_name, client_task_id, result: Shareable, fl_ctx: FLContext
+    ) -> None:
+        if self._phase == AppConstants.PHASE_TRAIN and task_name == self.train_task_name:
+            self._accept_train_result(client_name=client.name, result=result, fl_ctx=fl_ctx)
+            self.log_info(fl_ctx, f"Result of unknown task {task_name} sent to aggregator.")
+        else:
+            self.log_error(fl_ctx, "Ignoring result from unknown task.")
+
+    def _accept_train_result(self, client_name: str, result: Shareable, fl_ctx: FLContext) -> bool:
+
+        rc = result.get_return_code()
+        contribution_round = result.get_cookie(AppConstants.CONTRIBUTION_ROUND)
+        result.set_header(AppConstants.CONTRIBUTION_ROUND, contribution_round)
+
+        # Raise errors if bad peer context or execution exception.
+        if rc and rc != ReturnCode.OK:
+            if self.ignore_result_error:
+                self.log_error(f"Ignore the client train result. Train result error code: {rc}")
+                return False
+            else:
+                if rc in [ReturnCode.MISSING_PEER_CONTEXT, ReturnCode.BAD_PEER_CONTEXT]:
+                    self.system_panic("Peer context is bad or missing. ScatterAndGather exiting.", fl_ctx=fl_ctx)
+                    return False
+                elif rc in [ReturnCode.EXECUTION_EXCEPTION, ReturnCode.TASK_UNKNOWN]:
+                    self.system_panic(
+                        "Execution Exception in client training. ScatterAndGather exiting.", fl_ctx=fl_ctx
+                    )
+                    return False
+                elif rc in [
+                    ReturnCode.EXECUTION_RESULT_ERROR,
+                    ReturnCode.TASK_DATA_FILTER_ERROR,
+                    ReturnCode.TASK_RESULT_FILTER_ERROR,
+                ]:
+                    self.system_panic("Execution result is not a shareable. ScatterAndGather exiting.", fl_ctx=fl_ctx)
+                    return False
+
+        fl_ctx.set_prop(AppConstants.CURRENT_ROUND, self._current_round, private=True, sticky=False)
+        fl_ctx.set_prop(AppConstants.TRAINING_RESULT, result, private=True, sticky=False)
+        fl_ctx.set_prop(AppConstants.CONTRIBUTION_ROUND, contribution_round, private=True, sticky=False)
+        self.fire_event(AppEventType.BEFORE_CONTRIBUTION_ACCEPT, fl_ctx)
+
+        accepted = self.aggregator.accept(result, fl_ctx)
+        accepted_msg = "ACCEPTED" if accepted else "REJECTED"
+        self.log_info(fl_ctx, f"Contribution from {client_name} {accepted_msg} by the aggregator.")
+
+        fl_ctx.set_prop(AppConstants.AGGREGATION_ACCEPTED, accepted, private=True, sticky=False)
+        self.fire_event(AppEventType.AFTER_CONTRIBUTION_ACCEPT, fl_ctx)
+
+        return accepted
+
+    def _check_abort_signal(self, fl_ctx, abort_signal: Signal):
+        if abort_signal.triggered:
+            self._phase = AppConstants.PHASE_FINISHED
+            self.log_info(fl_ctx, f"Abort signal received. Exiting at round {self._current_round}.")
+            return True
+        return False
diff --git a/nvflare/fuel/__init__.py b/nvflare/fuel/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/fuel/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/fuel/common/__init__.py b/nvflare/fuel/common/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/fuel/common/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/fuel/common/ctx.py b/nvflare/fuel/common/ctx.py
new file mode 100644
index 0000000000..333daef4f7
--- /dev/null
+++ b/nvflare/fuel/common/ctx.py
@@ -0,0 +1,77 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+
+
+class SimpleContext(object):
+    """
+    A simple context containing a props dictionary of key value pairs and convenience methods.
+    """
+
+    def __init__(self):
+        self.props = {}
+
+    def set_prop(self, key, value):
+        self.props[key] = value
+
+    def set_props(self, props: dict):
+        if props:
+            self.props.update(props)
+
+    def len(self):
+        return len(self.props)
+
+    def get_prop(self, key, default=None):
+        return self.props.get(key, default)
+
+    def clear_props(self):
+        self.props = {}
+
+
+class BaseContext(SimpleContext):
+    """
+    A SimpleContext with threading locks.
+    """
+
+    def __init__(self):
+        SimpleContext.__init__(self)
+        self.asked_to_stop = False
+        self._update_lock = threading.Lock()
+
+    def ask_to_stop(self):
+        self.asked_to_stop = True
+
+    def set_prop(self, key, value):
+        with self._update_lock:
+            SimpleContext.set_prop(self, key, value)
+
+    def set_props(self, props: dict):
+        if not props:
+            return
+
+        with self._update_lock:
+            SimpleContext.set_props(self, props)
+
+    def len(self):
+        with self._update_lock:
+            return SimpleContext.len(self)
+
+    def get_prop(self, key, default=None):
+        with self._update_lock:
+            return SimpleContext.get_prop(self, key, default)
+
+    def clear_props(self):
+        with self._update_lock:
+            SimpleContext.clear_props(self)
diff --git a/nvflare/fuel/common/excepts.py b/nvflare/fuel/common/excepts.py
new file mode 100644
index 0000000000..1690290714
--- /dev/null
+++ b/nvflare/fuel/common/excepts.py
@@ -0,0 +1,21 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class ConfigError(Exception):
+    """
+    Raised when configuration parsing error happens.
+    """
+
+    pass
diff --git a/nvflare/fuel/common/multi_process_executor_constants.py b/nvflare/fuel/common/multi_process_executor_constants.py
new file mode 100644
index 0000000000..c0084c095f
--- /dev/null
+++ b/nvflare/fuel/common/multi_process_executor_constants.py
@@ -0,0 +1,39 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class CommunicationMetaData(object):
+    COMMAND = "command"
+    TASK_NAME = "task_name"
+    FL_CTX = "fl_ctx"
+    EVENT_TYPE = "event_type"
+    HANDLE_CONN = "handle_conn"
+    EXE_CONN = "exe_conn"
+    COMPONENTS = "MPExecutor_components"
+    HANDLERS = "MPExecutor_handlers"
+    LOCAL_EXECUTOR = "local_executor"
+    RANK_NUMBER = "rank_number"
+    SHAREABLE = "shareable"
+    RELAYER = "relayer"
+    RANK_PROCESS_STARTED = "rank_process_started"
+    PARENT_PASSWORD = "parent process secret password"
+    CHILD_PASSWORD = "client process secret password"
+
+
+class CommunicateData(object):
+    EXECUTE = "execute"
+    HANDLE_EVENT = "handle_event"
+    CLOSE = "close"
+    SUB_WORKER_PROCESS = "sub_worker_process"
+    MULTI_PROCESS_EXECUTOR = "multi_process_executor"
diff --git a/nvflare/fuel/hci/__init__.py b/nvflare/fuel/hci/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/fuel/hci/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/fuel/hci/base64_utils.py b/nvflare/fuel/hci/base64_utils.py
new file mode 100644
index 0000000000..7438bf0342
--- /dev/null
+++ b/nvflare/fuel/hci/base64_utils.py
@@ -0,0 +1,91 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+
+
+def bytes_to_b64str(data_bytes) -> str:
+    """Convert binary to base64-encoded string."""
+    encoded_bytes = base64.b64encode(data_bytes)
+    return encoded_bytes.decode("ascii")
+
+
+def b64str_to_bytes(b64str: str) -> bytes:
+    """Convert base64-encoded string to binary."""
+    encoded_bytes = b64str.encode("ascii")
+    return base64.b64decode(encoded_bytes)
+
+
+def binary_file_to_b64str(file_name) -> str:
+    """
+    Encode content of a binary file to a Base64-encoded ASCII string.
+
+    Args:
+        file_name: the binary file to be processed
+
+    Returns: base64-encoded ASCII string
+
+    """
+    data_bytes = open(file_name, "rb").read()
+    return bytes_to_b64str(data_bytes)
+
+
+def b64str_to_binary_file(b64str: str, file_name):
+    """
+    Decode a base64-encoded string and write it into a binary file.
+
+    Args:
+        b64str: the base64-encoded ASCII string
+        file_name: the file to write to
+
+    Returns: number of bytes written
+
+    """
+    data_bytes = b64str_to_bytes(b64str)
+    with open(file_name, "wb") as f:
+        f.write(data_bytes)
+    return len(data_bytes)
+
+
+def text_file_to_b64str(file_name) -> str:
+    """
+    Encode content of a text file to a Base64-encoded ASCII string.
+
+    Args:
+        file_name: name of the text file
+
+    Returns: base64-encoded string
+
+    """
+    data_string = open(file_name, "r").read()
+    data_bytes = data_string.encode("utf-8")
+    return bytes_to_b64str(data_bytes)
+
+
+def b64str_to_text_file(b64str: str, file_name):
+    """
+    Decode a base64-encoded string and write result into a text file.
+
+    Args:
+        b64str: base64-encoded string
+        file_name: file to be created
+
+    Returns: number of data types written (may not be the same as number of characters)
+
+    """
+    data_bytes = b64str_to_bytes(b64str)
+    data_string = data_bytes.decode("utf-8")
+    with open(file_name, "w") as f:
+        f.write(data_string)
+    return len(data_bytes)
diff --git a/nvflare/fuel/hci/client/__init__.py b/nvflare/fuel/hci/client/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/fuel/hci/client/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/fuel/hci/client/api.py b/nvflare/fuel/hci/client/api.py
new file mode 100644
index 0000000000..6583ca7fe0
--- /dev/null
+++ b/nvflare/fuel/hci/client/api.py
@@ -0,0 +1,451 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import socket
+import ssl
+import time
+import traceback
+from datetime import datetime
+
+from nvflare.fuel.hci.cmd_arg_utils import split_to_args
+from nvflare.fuel.hci.conn import Connection, receive_and_process
+from nvflare.fuel.hci.proto import make_error
+from nvflare.fuel.hci.reg import CommandModule, CommandRegister
+from nvflare.fuel.hci.security import get_certificate_common_name
+from nvflare.fuel.hci.table import Table
+
+from .api_status import APIStatus
+
+
+class ReplyProcessor(object):
+    """An abstract class for parsing server's response."""
+
+    def reply_start(self, client, reply_json):
+        pass
+
+    def process_string(self, client, item: str):
+        pass
+
+    def process_success(self, client, item: str):
+        pass
+
+    def process_error(self, client, err: str):
+        pass
+
+    def process_table(self, client, table: Table):
+        pass
+
+    def process_dict(self, client, d: dict):
+        pass
+
+    def process_shutdown(self, client, msg: str):
+        pass
+
+    def process_token(self, client, token: str):
+        pass
+
+    def protocol_error(self, client, err: str):
+        pass
+
+    def reply_done(self, client):
+        pass
+
+
+class _LoginReplyProcessor(ReplyProcessor):
+    """
+    Reply processor for handling login and setting the token for the admin client.
+    """
+
+    def process_string(self, client, item: str):
+        client.login_result = item
+
+    def process_token(self, client, token: str):
+        client.token = token
+
+
+class _CmdListReplyProcessor(ReplyProcessor):
+    """
+    Reply processor to register available commands after getting back a table of commands from the server.
+    """
+
+    def process_table(self, client, table: Table):
+        for i in range(len(table.rows)):
+            if i == 0:
+                # this is header
+                continue
+
+            row = table.rows[i]
+            if len(row) < 5:
+                return
+
+            scope = row[0]
+            cmd_name = row[1]
+            desc = row[2]
+            usage = row[3]
+            confirm = row[4]
+
+            # if confirm == 'auth' and not client.require_login:
+            # the user is not authenticated - skip this command
+            # continue
+
+            client.server_cmd_reg.add_command(
+                scope_name=scope,
+                cmd_name=cmd_name,
+                desc=desc,
+                usage=usage,
+                handler=None,
+                authz_func=None,
+                visible=True,
+                confirm=confirm,
+            )
+
+        client.server_cmd_received = True
+
+
+class AdminAPI:
+    """Underlying API to keep certs, keys and connection information and to execute admin commands through do_command.
+
+    Args:
+        host: cn provisioned for the project, with this fully qualified domain name resolving to the IP of the FL server
+        port: port provisioned as admin_port for FL admin communication, by default provisioned as 8003, must be int
+        ca_cert: path to CA Cert file, by default provisioned rootCA.pem
+        client_cert: path to admin client Cert file, by default provisioned as client.crt
+        client_key: path to admin client Key file, by default provisioned as client.key
+        upload_dir: File transfer upload directory. Folders uploaded to the server to be deployed must be here. Folder must already exist and be accessible.
+        download_dir: File transfer download directory. Can be same as upload_dir. Folder must already exist and be accessible.
+        server_cn: server cn (only used for validating server cn)
+        cmd_modules: command modules to load and register
+        poc: Whether to enable poc mode for using the proof of concept example without secure communication.
+        debug: Whether to print debug messages, which can help with diagnosing problems. False by default.
+    """
+
+    def __init__(
+        self,
+        host,
+        port,
+        ca_cert="",
+        client_cert="",
+        client_key="",
+        upload_dir="",
+        download_dir="",
+        server_cn=None,
+        cmd_modules=None,
+        poc=False,
+        debug=False,
+    ):
+        if cmd_modules is None:
+            from .file_transfer import FileTransferModule
+
+            cmd_modules = [FileTransferModule(upload_dir=upload_dir, download_dir=download_dir)]
+        self.host = host
+        self.port = port
+        self.poc = poc
+        if not self.poc:
+            if len(ca_cert) <= 0:
+                raise Exception("missing CA Cert file name")
+            self.ca_cert = ca_cert
+            if len(client_cert) <= 0:
+                raise Exception("missing Client Cert file name")
+            self.client_cert = client_cert
+            if len(client_key) <= 0:
+                raise Exception("missing Client Key file name")
+            self.client_key = client_key
+        self.server_cn = server_cn  # does not seem to be used right now
+        self.debug = debug
+
+        # for reply callbacks
+        self.reply_processor = None
+        self.command_result = None
+
+        # for login
+        self.token = None
+        self.login_result = None
+
+        self.server_cmd_reg = CommandRegister(app_ctx=self)
+        self.client_cmd_reg = CommandRegister(app_ctx=self)
+        self.server_cmd_received = False
+
+        self.shutdown_msg = None
+
+        self.all_cmds = []
+        self.iobuffer = None
+
+        self._load_client_cmds(cmd_modules)
+        if poc:
+            print("Please log in with login_with_password(username, password) to enable server cmds.")
+
+    def _load_client_cmds(self, cmd_modules):
+        if cmd_modules:
+            if not isinstance(cmd_modules, list):
+                raise TypeError("cmd_modules must be a list")
+            for m in cmd_modules:
+                if not isinstance(m, CommandModule):
+                    raise TypeError("cmd_modules must be a list of CommandModule")
+                self.client_cmd_reg.register_module(m, include_invisible=False)
+        self.client_cmd_reg.finalize(self.register_command)
+
+    def register_command(self, cmd_entry):
+        self.all_cmds.append(cmd_entry.name)
+
+    def logout(self):
+        """
+        Send logout command to server.
+        """
+        return self.server_execute("_logout")
+
+    def login(self, username: str):
+        """Login using certification files and retrieve server side commands.
+
+        Args:
+            username: Username
+
+        Returns:
+            Object containing status and details
+        """
+        self.login_result = None
+        self._try_command(f"_cert_login {username}", _LoginReplyProcessor())
+        if self.login_result is None:
+            return {"status": APIStatus.ERROR_RUNTIME, "details": "Communication Error - please try later"}
+        elif self.login_result == "REJECT":
+            return {"status": APIStatus.ERROR_CERT, "details": "Incorrect user name or certificate"}
+
+        # get command list from server
+        self.server_cmd_received = False
+        self._try_command("_commands", _CmdListReplyProcessor())
+        self.server_cmd_reg.finalize(self.register_command)
+        if not self.server_cmd_received:
+            return {"status": APIStatus.ERROR_RUNTIME, "details": "Communication Error - please try later"}
+
+        return {"status": APIStatus.SUCCESS, "details": "Login success"}
+
+    def login_with_password(self, username: str, password: str):
+        """Login using password for poc example.
+
+        Args:
+            username: Username
+            password: password
+
+        Returns:
+            Object containing status and details
+        """
+        self.login_result = None
+        self._try_command(f"_login {username} {password}", _LoginReplyProcessor())
+        if self.login_result is None:
+            return {"status": APIStatus.ERROR_RUNTIME, "details": "Communication Error - please try later"}
+        elif self.login_result == "REJECT":
+            return {"status": APIStatus.ERROR_CERT, "details": "Incorrect user name or certificate"}
+
+        # get command list from server
+        self.server_cmd_received = False
+        self._try_command("_commands", _CmdListReplyProcessor())
+        self.server_cmd_reg.finalize(self.register_command)
+        if not self.server_cmd_received:
+            return {"status": APIStatus.ERROR_RUNTIME, "details": "Communication Error - please try later"}
+
+        return {"status": APIStatus.SUCCESS, "details": "Login success"}
+
+    def _send_to_sock(self, sock, command, process_json_func):
+        conn = Connection(sock, self)
+        conn.append_command(command)
+        if self.token:
+            conn.append_token(self.token)
+
+        conn.close()
+        ok = receive_and_process(sock, process_json_func)
+        if not ok:
+            process_json_func(
+                make_error("Failed to communicate with Admin Server {} on {}".format(self.host, self.port))
+            )
+
+    def _process_server_reply(self, resp):
+        """Process the server reply and store the status/details into client's `command_result`
+
+        Args:
+            resp: The raw response that returns by the server.
+        """
+        if self.debug:
+            print("DEBUG: Server Reply: {}".format(resp))
+        self.command_result = resp  # this resp is what is usually directly used to return, straight from server
+        reply_processor = ReplyProcessor() if self.reply_processor is None else self.reply_processor
+
+        reply_processor.reply_start(self, resp)
+
+        if resp is not None:
+            data = resp["data"]
+            for item in data:
+                it = item["type"]
+                if it == "string":
+                    reply_processor.process_string(self, item["data"])
+                elif it == "success":
+                    reply_processor.process_success(self, item["data"])
+                elif it == "error":
+                    reply_processor.process_error(self, item["data"])
+                    break
+                elif it == "table":
+                    table = Table(None)
+                    table.set_rows(item["rows"])
+                    reply_processor.process_table(self, table)
+                elif it == "dict":
+                    reply_processor.process_dict(self, item["data"])
+                elif it == "token":
+                    reply_processor.process_token(self, item["data"])
+                elif it == "shutdown":
+                    reply_processor.process_shutdown(self, item["data"])
+                    break
+                else:
+                    reply_processor.protocol_error(self, "Invalid item type: " + it)
+                    break
+        else:
+            reply_processor.protocol_error(self, "Protocol Error")
+
+        reply_processor.reply_done(self)
+
+    def _try_command(self, command, reply_processor):
+        """Try to execute a command on server side.
+
+        Args:
+            command: The command to execute.
+            reply_processor: An instance of ReplyProcessor
+
+        """
+        # process_json_func can't return data because how "receive_and_process" is written.
+        self.reply_processor = reply_processor
+        process_json_func = self._process_server_reply
+        try:
+            if not self.poc:
+                # SSL communication
+                ctx = ssl.create_default_context()
+                ctx.verify_mode = ssl.CERT_REQUIRED
+                ctx.check_hostname = False
+
+                ctx.load_verify_locations(self.ca_cert)
+                ctx.load_cert_chain(certfile=self.client_cert, keyfile=self.client_key)
+
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+                    with ctx.wrap_socket(sock) as ssock:
+                        ssock.connect((self.host, self.port))
+                        if self.server_cn:
+                            # validate server CN
+                            cn = get_certificate_common_name(ssock.getpeercert())
+                            if cn != self.server_cn:
+                                process_json_func(
+                                    make_error("wrong server: expecting {} but connected {}".format(self.server_cn, cn))
+                                )
+                                return
+
+                        self._send_to_sock(ssock, command, process_json_func)
+            else:
+                # poc without certs
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+                    sock.connect((self.host, self.port))
+                    self._send_to_sock(sock, command, process_json_func)
+        except Exception as ex:
+            if self.debug:
+                traceback.print_exc()
+
+            process_json_func(
+                make_error("Failed to communicate with Admin Server {} on {}: {}".format(self.host, self.port, ex))
+            )
+
+    def do_command(self, command):
+        """A convenient method to call commands using string.
+
+        Args:
+          command (str): command
+
+        Returns:
+            Object containing status and details (or direct response from server, which originally was just time and data)
+        """
+        args = split_to_args(command)
+        cmd_name = args[0]
+        self.command_result = None
+
+        # check client side commands
+        entries = self.client_cmd_reg.get_command_entries(cmd_name)
+        if len(entries) > 1:
+            return {
+                "status": APIStatus.ERROR_SYNTAX,
+                "details": f"Ambiguous client command {cmd_name} - qualify with scope",
+            }
+        elif len(entries) == 1:
+            ent = entries[0]
+            ent.handler(args, self.client_cmd_reg.app_ctx)
+            if self.command_result is None:
+                return {"status": APIStatus.ERROR_RUNTIME, "details": "Client did not respond"}
+            return self.command_result
+
+        # check server side commands
+        entries = self.server_cmd_reg.get_command_entries(cmd_name)
+        if len(entries) <= 0:
+            return {
+                "status": APIStatus.ERROR_SYNTAX,
+                "details": f"Command {cmd_name} not found in server or client cmds",
+            }
+        elif len(entries) > 1:
+            return {
+                "status": APIStatus.ERROR_SYNTAX,
+                "details": f"Ambiguous server command {cmd_name} - qualify with scope",
+            }
+        return self.server_execute(command)
+
+    def server_execute(self, command, reply_processor=None):
+        """
+        This had to be kept relatively the same as in the hci Admin client with a wrapper to return, because the client
+        is passed to command_reg commands and reusing that like file_transfer, server_execute is directly called.
+        """
+        start = time.time()
+        self._try_command(command, reply_processor)
+        secs = time.time() - start
+        usecs = int(secs * 1000000)
+
+        if self.debug:
+            print(f"DEBUG: server_execute Done [{usecs} usecs] {datetime.now()}")
+
+        if self.command_result is None:
+            return {"status": APIStatus.ERROR_RUNTIME, "details": "Server did not respond"}
+        if "status" not in self.command_result:
+            self.command_result.update({"status": APIStatus.SUCCESS})
+        return self.command_result
+
+    def write_string(self, data: str):
+        content = data + "\n"
+        self.iobuffer.write(content)
+
+    def _show_one_command(self, cmd_name, reg):
+        entries = reg.get_command_entries(cmd_name)
+        if len(entries) <= 0:
+            self.write_string("Undefined command {}\n".format(cmd_name))
+            return
+
+        for e in entries:
+            if not e.visible:
+                continue
+
+            if len(e.scope.name) > 0:
+                self.write_string("Command: {}.{}".format(e.scope.name, cmd_name))
+            else:
+                self.write_string("Command: {}".format(cmd_name))
+
+            self.write_string("Description: {}".format(e.desc))
+            self.write_string("Usage: {}\n".format(e.usage))
+
+    def _show_commands(self, reg: CommandRegister):
+        table = Table(["Scope", "Command", "Description"])
+        for scope_name in sorted(reg.scopes):
+            scope = reg.scopes[scope_name]
+            for cmd_name in sorted(scope.entries):
+                e = scope.entries[cmd_name]
+                if e.visible:
+                    table.add_row([scope_name, cmd_name, e.desc])
+        table.write(self.iobuffer)
diff --git a/nvflare/fuel/hci/client/api_status.py b/nvflare/fuel/hci/client/api_status.py
new file mode 100644
index 0000000000..4dde2e46dc
--- /dev/null
+++ b/nvflare/fuel/hci/client/api_status.py
@@ -0,0 +1,32 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from enum import Enum
+
+
+class APIStatus(str, Enum):
+    """
+    Constants for the valid status options for the status of FLAdminAPIResponse.
+    """
+
+    SUCCESS = "SUCCESS"  # command issues successfully
+    ERROR_PROTOCOL = (
+        "ERROR_PROTOCOL"  # the payload/data is not following the correct format/protocol expected by the server
+    )
+    ERROR_CERT = "ERROR_CERT"  # key or certs are incorrect
+    ERROR_AUTHENTICATION = "ERROR_AUTHENTICATION"  # authentication failed, need to log in
+    ERROR_AUTHORIZATION = "ERROR_AUTHORIZATION"  # authorization failed, permissions
+    ERROR_SYNTAX = "ERROR_SYNTAX"  # command syntax incorrect
+    ERROR_RUNTIME = "ERROR_RUNTIME"  # various errors at runtime depending on the command
+    ERROR_INVALID_CLIENT = "ERROR_INVALID_CLIENT"  # wrong/invalid client names exists in command
diff --git a/nvflare/fuel/hci/client/file_transfer.py b/nvflare/fuel/hci/client/file_transfer.py
new file mode 100644
index 0000000000..6980c3dd77
--- /dev/null
+++ b/nvflare/fuel/hci/client/file_transfer.py
@@ -0,0 +1,276 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import traceback
+
+import nvflare.fuel.hci.file_transfer_defs as ftd
+from nvflare.fuel.hci.base64_utils import (
+    b64str_to_binary_file,
+    b64str_to_bytes,
+    b64str_to_text_file,
+    binary_file_to_b64str,
+    bytes_to_b64str,
+    text_file_to_b64str,
+)
+from nvflare.fuel.hci.cmd_arg_utils import join_args
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec
+from nvflare.fuel.hci.table import Table
+from nvflare.fuel.hci.zip_utils import unzip_all_from_bytes, zip_directory_to_bytes
+
+from .lib import AdminClient, ReplyProcessor
+
+
+def _server_cmd_name(name: str):
+    return ftd.SERVER_MODULE_NAME + "." + name
+
+
+class _DownloadProcessor(ReplyProcessor):
+    """Reply processor to handle downloads."""
+
+    def __init__(self, download_dir: str, str_to_file_func):
+        self.download_dir = download_dir
+        self.str_to_file_func = str_to_file_func
+        self.data_received = False
+        self.table = None
+
+    def reply_start(self, client, reply_json):
+        self.data_received = False
+        self.table = Table(["file", "size"])
+
+    def reply_done(self, client):
+        if not self.data_received:
+            client.write_error("protocol error - no data received")
+        else:
+            client.write_table(self.table)
+
+    def process_table(self, client, table: Table):
+        try:
+            rows = table.rows
+            if len(rows) < 1:
+                # no data
+                client.write_error("protocol error - no file data")
+                return
+
+            for i in range(len(rows)):
+                if i == 0:
+                    # this is header
+                    continue
+
+                row = rows[i]
+                if len(row) < 1:
+                    client.write_error("protocol error - missing file name")
+                    return
+
+                if len(row) < 2:
+                    client.write_error("protocol error - missing file data")
+                    return
+
+                file_name = row[0]
+                encoded_str = row[1]
+                full_path = os.path.join(self.download_dir, file_name)
+                num_bytes = self.str_to_file_func(encoded_str, full_path)
+                self.table.add_row([file_name, str(num_bytes)])
+                self.data_received = True
+        except Exception as ex:
+            traceback.print_exc()
+            client.write_error("exception processing file: {}".format(ex))
+
+
+class _DownloadFolderProcessor(ReplyProcessor):
+    """
+    Reply processor for handling downloading directories.
+    """
+
+    def __init__(self, download_dir: str):
+        self.download_dir = download_dir
+        self.data_received = False
+        self.table = None
+
+    def reply_start(self, client, reply_json):
+        self.data_received = False
+
+    def reply_done(self, client):
+        if not self.data_received:
+            client.write_error("protocol error - no data received")
+
+    def process_string(self, client, item: str):
+        try:
+            self.data_received = True
+            data_bytes = b64str_to_bytes(item)
+            unzip_all_from_bytes(data_bytes, self.download_dir)
+        except Exception as ex:
+            traceback.print_exc()
+            client.write_error("exception processing reply: {}".format(ex))
+
+
+class FileTransferModule(CommandModule):
+    """
+    Command module with commands relevant to file transfer.
+    """
+
+    def __init__(self, upload_dir: str, download_dir: str, upload_folder_cmd_name="upload_app"):
+        assert os.path.isdir(upload_dir), "upload_dir {} is not a valid dir".format(upload_dir)
+
+        assert os.path.isdir(download_dir), "download_dir {} is not a valid dir".format(download_dir)
+
+        self.upload_dir = upload_dir
+        self.download_dir = download_dir
+        self.upload_folder_cmd_name = upload_folder_cmd_name
+
+    def get_spec(self):
+        return CommandModuleSpec(
+            name="file_transfer",
+            cmd_specs=[
+                CommandSpec(
+                    name="upload_text",
+                    description="upload one or more text files in the upload_dir",
+                    usage="upload_text file_name ...",
+                    handler_func=self.upload_text_file,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name="download_text",
+                    description="download one or more text files in the download_dir",
+                    usage="download_text file_name ...",
+                    handler_func=self.download_text_file,
+                ),
+                CommandSpec(
+                    name="upload_binary",
+                    description="upload one or more binary files in the upload_dir",
+                    usage="upload_binary file_name ...",
+                    handler_func=self.upload_binary_file,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name="download_binary",
+                    description="download one or more binary files in the download_dir",
+                    usage="download_binary file_name ...",
+                    handler_func=self.download_binary_file,
+                ),
+                CommandSpec(
+                    name=self.upload_folder_cmd_name,
+                    description="upload application to the server",
+                    usage=self.upload_folder_cmd_name + " application_folder",
+                    handler_func=self.upload_folder,
+                ),
+                CommandSpec(
+                    name="download_folder",
+                    description="download a folder from the server",
+                    usage="download_folder folder_name",
+                    handler_func=self.download_folder,
+                ),
+                CommandSpec(
+                    name="info",
+                    description="show folder setup info",
+                    usage="info",
+                    handler_func=self.info,
+                ),
+            ],
+        )
+
+    def upload_file(self, args, admin: AdminClient, cmd_name, file_to_str_func):
+        full_cmd_name = _server_cmd_name(cmd_name)
+        if len(args) < 2:
+            admin.write_error("syntax error: missing file names")
+            return
+
+        parts = [full_cmd_name]
+        for i in range(1, len(args)):
+            file_name = args[i]
+            full_path = os.path.join(self.upload_dir, file_name)
+            if not os.path.isfile(full_path):
+                admin.write_error("no such file: {}".format(full_path))
+                return
+
+            encoded_string = file_to_str_func(full_path)
+            parts.append(file_name)
+            parts.append(encoded_string)
+
+        command = join_args(parts)
+        admin.server_execute(command)
+
+    def upload_text_file(self, args, admin: AdminClient):
+        self.upload_file(args, admin, ftd.SERVER_CMD_UPLOAD_TEXT, text_file_to_b64str)
+
+    def upload_binary_file(self, args, admin: AdminClient):
+        self.upload_file(args, admin, ftd.SERVER_CMD_UPLOAD_BINARY, binary_file_to_b64str)
+
+    def download_file(self, args, admin: AdminClient, cmd_name, str_to_file_func):
+        full_cmd_name = _server_cmd_name(cmd_name)
+        if len(args) < 2:
+            admin.write_error("syntax error: missing file names")
+            return
+
+        parts = [full_cmd_name]
+        for i in range(1, len(args)):
+            file_name = args[i]
+            parts.append(file_name)
+
+        command = join_args(parts)
+        reply_processor = _DownloadProcessor(self.download_dir, str_to_file_func)
+        admin.server_execute(command, reply_processor)
+
+    def download_text_file(self, args, admin: AdminClient):
+        self.download_file(args, admin, ftd.SERVER_CMD_DOWNLOAD_TEXT, b64str_to_text_file)
+
+    def download_binary_file(self, args, admin: AdminClient):
+        self.download_file(args, admin, ftd.SERVER_CMD_DOWNLOAD_BINARY, b64str_to_binary_file)
+
+    def upload_folder(self, args, admin: AdminClient):
+        if len(args) != 2:
+            admin.write_error("usage: " + self.upload_folder_cmd_name + " folder_name")
+            return
+
+        folder_name = args[1]
+        if folder_name.endswith("/"):
+            folder_name = folder_name.rstrip("/")
+
+        full_path = os.path.join(self.upload_dir, folder_name)
+        if not os.path.isdir(full_path):
+            admin.write_error('"{}" is not a valid folder'.format(full_path))
+            return
+
+        # zip the data
+        data = zip_directory_to_bytes(self.upload_dir, folder_name)
+
+        # prepare for upload
+        folder_name = os.path.split(folder_name)[1]
+        rel_path = os.path.relpath(full_path, self.upload_dir)
+        folder_name = self._remove_loading_dotdot(rel_path)
+
+        b64str = bytes_to_b64str(data)
+        parts = [_server_cmd_name(ftd.SERVER_CMD_UPLOAD_FOLDER), folder_name, b64str]
+        command = join_args(parts)
+        admin.server_execute(command)
+
+    def download_folder(self, args, admin: AdminClient):
+        if len(args) != 2:
+            admin.write_error("usage: download_folder folder_name")
+            return
+
+        parts = [_server_cmd_name(ftd.SERVER_CMD_DOWNLOAD_FOLDER), args[1]]
+        command = join_args(parts)
+        reply_processor = _DownloadFolderProcessor(self.download_dir)
+        admin.server_execute(command, reply_processor)
+
+    def info(self, args, admin: AdminClient):
+        admin.write_string("Local Upload Source: {}".format(self.upload_dir))
+        admin.write_string("Local Download Destination: {}".format(self.download_dir))
+        admin.server_execute(_server_cmd_name(ftd.SERVER_CMD_INFO))
+
+    def _remove_loading_dotdot(self, path):
+        while path.startswith("../"):
+            path = path[3:]
+        return path
diff --git a/nvflare/fuel/hci/client/fl_admin_api.py b/nvflare/fuel/hci/client/fl_admin_api.py
new file mode 100644
index 0000000000..b05e1e235c
--- /dev/null
+++ b/nvflare/fuel/hci/client/fl_admin_api.py
@@ -0,0 +1,879 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import time
+from functools import wraps
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from nvflare.fuel.hci.client.api import AdminAPI
+from nvflare.fuel.hci.client.api_status import APIStatus
+from nvflare.fuel.hci.client.fl_admin_api_constants import FLDetailKey
+from nvflare.fuel.hci.client.fl_admin_api_spec import APISyntaxError, FLAdminAPIResponse, FLAdminAPISpec, TargetType
+
+
+def wrap_with_return_exception_responses(func):
+    """Decorator on all FLAdminAPI calls to handle any raised exceptions and return the fitting error status."""
+
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        try:
+            reply = func(self, *args, **kwargs)
+            if reply:
+                return reply
+            else:
+                return FLAdminAPIResponse(
+                    APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not generate reply."}
+                )
+        except ConnectionRefusedError as e:
+            return FLAdminAPIResponse(APIStatus.ERROR_AUTHENTICATION, {"message": "Error: " + str(e)})
+        except PermissionError as e:
+            return FLAdminAPIResponse(APIStatus.ERROR_AUTHORIZATION, {"message": "Error: " + str(e)})
+        except LookupError as e:
+            return FLAdminAPIResponse(APIStatus.ERROR_INVALID_CLIENT, {"message": "Error: " + str(e)})
+        except APISyntaxError as e:
+            return FLAdminAPIResponse(APIStatus.ERROR_SYNTAX, {"message": "Error: " + str(e)})
+        except TimeoutError as e:
+            return FLAdminAPIResponse(
+                APIStatus.ERROR_RUNTIME,
+                {"message": "TimeoutError: possibly unable to communicate with server. " + str(e)},
+            )
+        except Exception as e:
+            return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": "Exception: " + str(e)})
+
+    return wrapper
+
+
+def default_server_status_handling_cb(reply: FLAdminAPIResponse, **kwargs) -> bool:
+    if reply["details"][FLDetailKey.SERVER_ENGINE_STATUS] == "stopped":
+        return True
+    else:
+        return False
+
+
+def default_client_status_handling_cb(reply: FLAdminAPIResponse) -> bool:
+    client_statuses = reply.get("details").get("client_statuses")
+    stopped_client_count = 0
+    for i in range(1, len(client_statuses)):
+        if client_statuses[i][3] == "stopped":
+            stopped_client_count = stopped_client_count + 1
+    if stopped_client_count == len(client_statuses) - 1:
+        return True
+    else:
+        return False
+
+
+def default_stats_handling_cb(reply: FLAdminAPIResponse) -> bool:
+    if reply.get("details").get("message").get("ServerRunner").get("status") == "done":
+        return True
+    else:
+        return False
+
+
+class FLAdminAPI(AdminAPI, FLAdminAPISpec):
+    def __init__(
+        self,
+        host,
+        port,
+        ca_cert="",
+        client_cert="",
+        client_key="",
+        upload_dir="",
+        download_dir="",
+        poc=False,
+        debug=False,
+    ):
+        """
+        Initializes FLAdminAPI with required parameters to serve as foundation for communications to FL server.
+        Inherited AdminAPI's login(username) function can be called with the username of the admin corresponding to the
+        cert and key files to enable the usage of FLAdminAPI's functions.
+        Args:
+            host: cn provisioned for the project, with this fully qualified domain name resolving to the IP of the FL server
+            port: port provisioned as admin_port for FL admin communication, by default provisioned as 8003, must be int
+            ca_cert: path to CA Cert file, by default provisioned rootCA.pem
+            client_cert: path to admin client Cert file, by default provisioned as client.crt
+            client_key: path to admin client Key file, by default provisioned as client.key
+            upload_dir: File transfer upload directory. Folders uploaded to the server to be deployed must be here. Folder must already exist and be accessible.
+            download_dir: File transfer download directory. Can be same as upload_dir. Folder must already exist and be accessible.
+            poc: Whether to enable poc mode for using the proof of concept example without secure communication.
+            debug: Whether to print debug messages. False by default.
+        """
+        super().__init__(
+            host=host,
+            port=port,
+            ca_cert=ca_cert,
+            client_cert=client_cert,
+            client_key=client_key,
+            upload_dir=upload_dir,
+            download_dir=download_dir,
+            poc=poc,
+            debug=debug,
+        )
+        self.upload_dir = upload_dir
+        self.download_dir = download_dir
+        self._error_buffer = None
+
+    def _process_targets_into_str(self, targets: List[str]) -> str:
+        if not isinstance(targets, list):
+            raise APISyntaxError("targets is not a list.")
+        if not all(isinstance(t, str) for t in targets):
+            raise APISyntaxError("all targets in the list of targets must be strings.")
+        for t in targets:
+            try:
+                self._validate_required_target_string(t)
+            except APISyntaxError:
+                raise APISyntaxError("each target in targets must be a string of only valid characters and no spaces.")
+        return " ".join(targets)
+
+    def _validate_required_target_string(self, target: str) -> str:
+        """Returns the target string if it exists and is valid."""
+        if not target:
+            raise APISyntaxError("target is required but not specified.")
+        if not isinstance(target, str):
+            raise APISyntaxError("target is not str.")
+        if not re.match("^[A-Za-z0-9._-]*$", target):
+            raise APISyntaxError("target must be a string of only valid characters and no spaces.")
+        return target
+
+    def _validate_options_string(self, options: str) -> str:
+        """Returns the options string if it is valid."""
+        if not isinstance(options, str):
+            raise APISyntaxError("options is not str.")
+        if not re.match("^[A-Za-z0-9- ]*$", options):
+            raise APISyntaxError("options must be a string of only valid characters.")
+        return options
+
+    def _validate_path_string(self, path: str) -> str:
+        """Returns the path string if it is valid."""
+        if not isinstance(path, str):
+            raise APISyntaxError("path is not str.")
+        if not re.match("^[A-Za-z0-9-._/]*$", path):
+            raise APISyntaxError("unsupported characters in path {}".format(path))
+        if path.startswith("/"):
+            raise APISyntaxError("absolute path is not allowed")
+        paths = path.split("/")
+        for p in paths:
+            if p == "..":
+                raise APISyntaxError(".. in path name is not allowed")
+        return path
+
+    def _validate_file_string(self, file: str) -> str:
+        """Returns the file string if it is valid."""
+        if not isinstance(file, str):
+            raise APISyntaxError("file is not str.")
+        if not re.match("^[A-Za-z0-9-._/]*$", file):
+            raise APISyntaxError("unsupported characters in file {}".format(file))
+        if file.startswith("/"):
+            raise APISyntaxError("absolute path for file is not allowed")
+        paths = file.split("/")
+        for p in paths:
+            if p == "..":
+                raise APISyntaxError(".. in file path is not allowed")
+        basename, file_extension = os.path.splitext(file)
+        if file_extension not in [".txt", ".log", ".json", ".csv", ".sh", ".config", ".py"]:
+            raise APISyntaxError(
+                "this command cannot be applied to file {}. Only files with the following extensions are "
+                "permitted: .txt, .log, .json, .csv, .sh, .config, .py".format(file)
+            )
+        return file
+
+    def _get_processed_cmd_reply_data(self, command) -> Tuple[bool, str, Dict[str, Any]]:
+        """Executes the specified command through the underlying AdminAPI's do_command() and checks the response to
+        raise common errors.
+
+        Returns:
+            Tuple of bool to indicate if success is in reply data, str with full response of the reply data, and the raw
+            reply.
+        """
+        success_in_data = False
+        reply = self.do_command(command)
+        # handle errors from write_error (these can be from FileTransferModule)
+        if self._error_buffer:
+            err = self._error_buffer
+            self._error_buffer = None
+            raise RuntimeError(err)
+        if reply.get("status") != APIStatus.SUCCESS:
+            raise RuntimeError(reply.get("details"))
+        reply_data_list = []
+        reply_data_full_response = ""
+        if reply.get("data"):
+            for data in reply["data"]:
+                if data["type"] == "success":
+                    success_in_data = True
+                if data["type"] == "string" or data["type"] == "error":
+                    reply_data_list.append(data["data"])
+            reply_data_full_response = "\n".join(reply_data_list)
+            if "not authenticated - no user" in reply_data_full_response:
+                raise ConnectionRefusedError(reply_data_full_response)
+            if "Failed to communicate" in reply_data_full_response:
+                raise ConnectionError(reply_data_full_response)
+            if "invalid client" in reply_data_full_response:
+                raise LookupError(reply_data_full_response)
+            if "unknown site" in reply_data_full_response:
+                raise LookupError(reply_data_full_response)
+            if "Authorization Error" in reply_data_full_response:
+                raise PermissionError(reply_data_full_response)
+        return success_in_data, reply_data_full_response, reply
+
+    def _parse_section_of_response_text(
+        self, data, start_string: str, offset: int = None, end_string: str = None, end_index=None
+    ) -> str:
+        """Convenience method to get portion of string based on parameters."""
+        if not offset:
+            offset = len(start_string) + 1
+        if end_string:
+            return data[data.find(start_string) + offset : data.find(end_string)]
+        if end_index:
+            return data[data.find(start_string) + offset : end_index]
+        return data[data.find(start_string) + offset :]
+
+    def _parse_section_of_response_text_as_int(
+        self, data, start_string: str, offset: int = None, end_string: str = None, end_index=None
+    ) -> int:
+        try:
+            return int(
+                self._parse_section_of_response_text(
+                    data=data, start_string=start_string, offset=offset, end_string=end_string, end_index=end_index
+                )
+            )
+        except ValueError:
+            return -1
+
+    def write_error(self, error: str) -> None:
+        """Internally used to handle errors from FileTransferModule"""
+        self._error_buffer = error
+
+    @wrap_with_return_exception_responses
+    def check_status(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        if target_type == TargetType.SERVER:
+            return self._check_status_server()
+        elif target_type == TargetType.CLIENT:
+            return self._check_status_client(targets)
+        else:
+            raise APISyntaxError("target_type must be server or client.")
+
+    def _check_status_server(self) -> FLAdminAPIResponse:
+        """
+        Checks the server status and returns the details. This call does not wait for the server to retrieve information
+        on the clients but returns the last information the server had at the time this call is made.
+
+        """
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data("check_status server")
+        details = {}
+        if reply.get("data"):
+            for data in reply["data"]:
+                if data["type"] == "string":
+                    if data["data"].find("Engine status:") != -1:
+                        details[FLDetailKey.SERVER_ENGINE_STATUS] = self._parse_section_of_response_text(
+                            data=data["data"], start_string="Engine status:"
+                        )
+                    if data["data"].find("Current run number:") != -1:
+                        details[FLDetailKey.RUN_NUMBER] = self._parse_section_of_response_text(
+                            data=data["data"], start_string="Current run number:"
+                        )
+                    if data["data"].find("FL_app name:") != -1:
+                        details[FLDetailKey.APP_NAME] = self._parse_section_of_response_text(
+                            data=data["data"], start_string="FL_app name:"
+                        )
+                    if data["data"].find("Registered clients:") != -1:
+                        details[FLDetailKey.REGISTERED_CLIENTS] = self._parse_section_of_response_text_as_int(
+                            data=data["data"], start_string="Registered clients:"
+                        )
+                if data["type"] == "table":
+                    details[FLDetailKey.STATUS_TABLE] = data["rows"]
+            return FLAdminAPIResponse(APIStatus.SUCCESS, details, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    def _check_status_client(self, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        if targets:
+            processed_targets_str = self._process_targets_into_str(targets)
+            command = "check_status client " + processed_targets_str
+        else:
+            command = "check_status client"
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        details = {}
+        if reply.get("data"):
+            for data in reply["data"]:
+                if data["type"] == "table":
+                    details["client_statuses"] = data["rows"]
+            return FLAdminAPIResponse(APIStatus.SUCCESS, details, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def set_run_number(self, run_number: int) -> FLAdminAPIResponse:
+        if not isinstance(run_number, int):
+            raise APISyntaxError("run_number must be int.")
+        if run_number < 1:
+            raise APISyntaxError("run_number must be int > 0.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(
+            "set_run_number " + str(run_number)
+        )
+        if reply_data_full_response:
+            if "Created a new run folder" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response})
+            if "run number already exist" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response})
+            if "needs to be an integer" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_SYNTAX, {"message": reply_data_full_response})
+            if "run_number can not be changed" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response})
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def delete_run_number(self, run_number: int) -> FLAdminAPIResponse:
+        if not isinstance(run_number, int):
+            raise APISyntaxError("run_number must be int.")
+        if run_number < 1:
+            raise APISyntaxError("run_number must be int > 0.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(
+            "delete_run_number " + str(run_number)
+        )
+        if reply_data_full_response:
+            if "needs to be an integer" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_SYNTAX, {"message": reply_data_full_response})
+            if "Current running run_number can not be deleted" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response})
+        if success:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response}, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def upload_app(self, app: str) -> FLAdminAPIResponse:
+        if not app:
+            raise APISyntaxError("app is required but not specified.")
+        if not isinstance(app, str):
+            raise APISyntaxError("app must be str.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data("upload_app " + app)
+        if reply_data_full_response:
+            if "Created folder" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response})
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def deploy_app(self, app: str, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        if not app:
+            raise APISyntaxError("app is required but not specified.")
+        if not isinstance(app, str):
+            raise APISyntaxError("app must be str.")
+        if target_type == TargetType.ALL:
+            command = "deploy_app " + app + " all"
+        elif target_type == TargetType.SERVER:
+            command = "deploy_app " + app + " server"
+        elif target_type == TargetType.CLIENT:
+            if targets:
+                processed_targets_str = self._process_targets_into_str(targets)
+                command = "deploy_app " + app + " client " + processed_targets_str
+            else:
+                command = "deploy_app " + app + " client"
+        else:
+            raise APISyntaxError("target_type must be server, client, or all.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            if "does not exist" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response}, reply)
+            if "Please set a run number" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response}, reply)
+        if success:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response}, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def start_app(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        if target_type == TargetType.ALL:
+            command = "start_app all"
+        elif target_type == TargetType.SERVER:
+            command = "start_app server"
+        elif target_type == TargetType.CLIENT:
+            if targets:
+                processed_targets_str = self._process_targets_into_str(targets)
+                command = "start_app client " + processed_targets_str
+            else:
+                command = "start_app client"
+        else:
+            raise APISyntaxError("target_type must be server, client, or all.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            if "Server already starting or started" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response}, reply)
+            if "Please set a run number" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response}, reply)
+        if success:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response}, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def abort(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        if target_type == TargetType.ALL:
+            command = "abort all"
+        elif target_type == TargetType.SERVER:
+            command = "abort server"
+        elif target_type == TargetType.CLIENT:
+            if targets:
+                processed_targets_str = self._process_targets_into_str(targets)
+                command = "abort client " + processed_targets_str
+            else:
+                command = "abort client"
+        else:
+            raise APISyntaxError("target_type must be server, client, or all.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            if "Server app has not started" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response}, reply)
+            if "No clients to abort" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response}, reply)
+            if "please wait for started before abort" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response}, reply)
+        if success:
+            return_details = {}
+            if reply_data_full_response:
+                return_details["message"] = reply_data_full_response
+            if reply.get("data"):
+                for data in reply["data"]:
+                    if data["type"] == "table":
+                        return_details[FLDetailKey.RESPONSES] = data["rows"]
+            return FLAdminAPIResponse(APIStatus.SUCCESS, return_details, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def restart(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        if target_type == TargetType.ALL:
+            command = "restart all"
+        elif target_type == TargetType.SERVER:
+            command = "restart server"
+        elif target_type == TargetType.CLIENT:
+            if targets:
+                processed_targets_str = self._process_targets_into_str(targets)
+                command = "restart client " + processed_targets_str
+            else:
+                command = "restart client"
+        else:
+            raise APISyntaxError("target_type must be server, client, or all.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            if "no clients available" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response})
+            if "Server is starting, please wait for started before restart" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response})
+        if success:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response}, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def shutdown(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        if target_type == TargetType.ALL:
+            command = "shutdown all"
+        elif target_type == TargetType.SERVER:
+            command = "shutdown server"
+        elif target_type == TargetType.CLIENT:
+            if targets:
+                processed_targets_str = self._process_targets_into_str(targets)
+                command = "shutdown client " + processed_targets_str
+            else:
+                command = "shutdown client"
+        else:
+            raise APISyntaxError("target_type must be server, client, or all.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            if "There are still active clients. Shutdown all clients first." in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response})
+            if "no clients to shutdown" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response})
+            if "Server is starting, please wait for started before shutdown" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response})
+        if success:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response}, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def remove_client(self, targets: List[str]) -> FLAdminAPIResponse:
+        if not targets:
+            raise APISyntaxError("targets needs to be provided as a list of client names.")
+        processed_targets_str = self._process_targets_into_str(targets)
+        command = "remove_client " + processed_targets_str
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if success:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response}, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def set_timeout(self, timeout: float) -> FLAdminAPIResponse:
+        if not isinstance(timeout, (float, int)):
+            raise APISyntaxError("timeout is not float.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data("set_timeout " + str(timeout))
+        if success:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response}, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def get_available_apps_to_upload(self):
+        dir_list = []
+        for item in os.listdir(self.upload_dir):
+            if os.path.isdir(os.path.join(self.upload_dir, item)):
+                dir_list.append(item)
+        return FLAdminAPIResponse(APIStatus.SUCCESS, {"app_list": dir_list})
+
+    @wrap_with_return_exception_responses
+    def ls_target(self, target: str, options: str = None, path: str = None) -> FLAdminAPIResponse:
+        target = self._validate_required_target_string(target)
+        command = "ls " + target
+        if options:
+            options = self._validate_options_string(options)
+            command = command + " " + options
+        if path:
+            path = self._validate_path_string(path)
+            command = command + " " + path
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response})
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def cat_target(self, target: str, options: str = None, file: str = None) -> FLAdminAPIResponse:
+        if not file:
+            raise APISyntaxError("file is required but not specified.")
+        file = self._validate_file_string(file)
+        target = self._validate_required_target_string(target)
+        command = "cat " + target
+        if options:
+            options = self._validate_options_string(options)
+            command = command + " " + options
+        if file:
+            command = command + " " + file
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response})
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def tail_target_log(self, target: str, options: str = None) -> FLAdminAPIResponse:
+        target = self._validate_required_target_string(target)
+        command = "tail " + target
+        if options:
+            options = self._validate_options_string(options)
+            command = command + " " + options
+        command = command + " log.txt"
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response})
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def env_target(self, target: str) -> FLAdminAPIResponse:
+        target = self._validate_required_target_string(target)
+        command = "env " + target
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            details = {}
+            environment = reply_data_full_response.split("\n")
+            for e in environment:
+                # set key and value to contents of each line with first = as separator
+                details[e[0 : e.find("=")]] = e[e.find("=") + 1 :]
+            return FLAdminAPIResponse(APIStatus.SUCCESS, details)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def get_working_directory(self, target: str) -> FLAdminAPIResponse:
+        target = self._validate_required_target_string(target)
+        command = "pwd " + target
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response})
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def grep_target(
+        self, target: str, options: str = None, pattern: str = None, file: str = None
+    ) -> FLAdminAPIResponse:
+        if not file:
+            raise APISyntaxError("file is required but not specified.")
+        file = self._validate_file_string(file)
+        if not pattern:
+            raise APISyntaxError("pattern is required but not specified.")
+        if not isinstance(pattern, str):
+            raise APISyntaxError("pattern is not str.")
+        target = self._validate_required_target_string(target)
+        command = "grep " + target
+        if options:
+            options = self._validate_options_string(options)
+            command = command + " " + options
+        command = command + " " + pattern + " " + file
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply_data_full_response:
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response})
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def show_stats(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        if target_type == TargetType.SERVER:
+            command = "show_stats server"
+        elif target_type == TargetType.CLIENT:
+            if targets:
+                processed_targets_str = self._process_targets_into_str(targets)
+                command = "show_stats client " + processed_targets_str
+            else:
+                command = "show_stats client"
+        else:
+            raise APISyntaxError("target_type must be server or client.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply.get("data"):
+            for data in reply["data"]:
+                if data["type"] == "dict":
+                    stats_result = data["data"]
+                    return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": stats_result}, reply)
+        if reply_data_full_response:
+            if "App is not running" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response}, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def show_errors(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        if target_type == TargetType.SERVER:
+            command = "show_errors server"
+        elif target_type == TargetType.CLIENT:
+            if targets:
+                processed_targets_str = self._process_targets_into_str(targets)
+                command = "show_errors client " + processed_targets_str
+            else:
+                command = "show_errors client"
+        else:
+            raise APISyntaxError("target_type must be server or client.")
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data(command)
+        if reply.get("data"):
+            for data in reply["data"]:
+                if data["type"] == "dict":
+                    errors_result = data["data"]
+                    return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": errors_result}, reply)
+        if reply_data_full_response:
+            if "App is not running" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response}, reply)
+        return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": "No errors."}, reply)
+
+    @wrap_with_return_exception_responses
+    def reset_errors(self) -> FLAdminAPIResponse:
+        success, reply_data_full_response, reply = self._get_processed_cmd_reply_data("reset_errors")
+        if reply_data_full_response:
+            if "App is not running" in reply_data_full_response:
+                return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": reply_data_full_response}, reply)
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": reply_data_full_response}, reply)
+        return FLAdminAPIResponse(
+            APIStatus.ERROR_RUNTIME, {"message": "Runtime error: could not handle server reply."}, reply
+        )
+
+    @wrap_with_return_exception_responses
+    def get_connected_client_list(self) -> FLAdminAPIResponse:
+        reply = self._check_status_server()
+        if reply["status"] == APIStatus.SUCCESS:
+            status_table = reply["details"][FLDetailKey.STATUS_TABLE]
+            list_of_connected_clients = []
+            for row in status_table:
+                if row[0] != "CLIENT NAME":
+                    list_of_connected_clients.append(row[0])
+            return FLAdminAPIResponse(APIStatus.SUCCESS, {FLDetailKey.CONNECTED_CLIENTS: list_of_connected_clients})
+        else:
+            return FLAdminAPIResponse(APIStatus.ERROR_RUNTIME, {"message": "runtime error"}, reply)
+
+    @wrap_with_return_exception_responses
+    def wait_until_server_status(
+        self,
+        interval: int = 20,
+        timeout: int = None,
+        callback: Callable[[FLAdminAPIResponse, Optional[List]], bool] = default_server_status_handling_cb,
+        fail_attempts: int = 3,
+        **kwargs,
+    ) -> FLAdminAPIResponse:
+        failed_attempts = 0
+        start = time.time()
+        while True:
+            reply = self._check_status_server()
+            if reply["details"].get(FLDetailKey.SERVER_ENGINE_STATUS):
+                met = callback(reply, **kwargs)
+                if met:
+                    return FLAdminAPIResponse(APIStatus.SUCCESS, {}, None)
+                fail_attempts = 0
+            else:
+                print("Could not get reply from check status server, trying again later")
+                failed_attempts += 1
+
+            now = time.time()
+            if timeout is not None:
+                if now - start >= timeout:
+                    return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": "Waited until timeout."}, None)
+            if failed_attempts > fail_attempts:
+                return FLAdminAPIResponse(
+                    APIStatus.ERROR_RUNTIME,
+                    {
+                        "message": "FL server status was not obtainable for more than the specified number of "
+                        "fail_attempts. "
+                    },
+                    None,
+                )
+            time.sleep(interval)
+
+    @wrap_with_return_exception_responses
+    def wait_until_client_status(
+        self,
+        interval: int = 10,
+        timeout: int = None,
+        callback: Callable[[FLAdminAPIResponse, Optional[List]], bool] = default_client_status_handling_cb,
+        fail_attempts: int = 6,
+        **kwargs,
+    ) -> FLAdminAPIResponse:
+        """This is similar to wait_until_server_status() and is an example for using other information from a repeated
+        call, in this case check_status(TargetType.CLIENT). Custom code can be written to use any data available from
+        any call to make decisions for how to proceed. Take caution that the conditions will be met at some point, or
+        timeout should be set with logic outside this function to handle checks for potential errors or this may loop
+        indefinitely.
+
+        Args:
+            interval: in seconds, the time between consecutive checks of the server
+            timeout: if set, the amount of time this function will run until before returning a response message
+            callback: the reply from show_stats(TargetType.SERVER) will be passed to the callback, along with any additional kwargs
+            which can go on to perform additional logic.
+            fail_attempts: number of consecutive failed attempts of getting the server status before returning with ERROR_RUNTIME.
+
+        Returns: FLAdminAPIResponse
+
+        """
+        failed_attempts = 0
+        start = time.time()
+        while True:
+            try:
+                reply = self.check_status(TargetType.CLIENT)
+                if reply:
+                    met = callback(reply, **kwargs)
+                    if met:
+                        return FLAdminAPIResponse(APIStatus.SUCCESS, {}, None)
+                    fail_attempts = 0
+                else:
+                    print("Could not get reply from check status client, trying again later")
+                    failed_attempts += 1
+            except BaseException as e:
+                print("Could not get clients stats, trying again later. Exception: ", e)
+                failed_attempts += 1
+
+            now = time.time()
+            if timeout is not None:
+                if now - start >= timeout:
+                    return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": "Waited until timeout."}, None)
+            if failed_attempts > fail_attempts:
+                return FLAdminAPIResponse(
+                    APIStatus.ERROR_RUNTIME,
+                    {
+                        "message": "FL client status was not obtainable for more than the specified number of "
+                        "fail_attempts. "
+                    },
+                    None,
+                )
+            time.sleep(interval)
+
+    @wrap_with_return_exception_responses
+    def wait_until_server_stats(
+        self,
+        interval: int = 10,
+        timeout: int = None,
+        callback: Callable[[FLAdminAPIResponse, Optional[List]], bool] = default_stats_handling_cb,
+        fail_attempts: int = 6,
+        **kwargs,
+    ) -> FLAdminAPIResponse:
+        """This is similar to wait_until_server_status() and is an example for using other information from a repeated
+        call, in this case show_stats(TargetType.SERVER). Custom code can be written to use any data available from any
+        call to make decisions for how to proceed. Take caution that the conditions will be met at some point, or
+        timeout should be set with logic outside this function to handle checks for potential errors or this may loop
+        indefinitely.
+
+        Args:
+            interval: in seconds, the time between consecutive checks of the server
+            timeout: if set, the amount of time this function will run until before returning a response message
+            callback: the reply from show_stats(TargetType.SERVER) will be passed to the callback, along with any additional kwargs
+            which can go on to perform additional logic.
+            fail_attempts: number of consecutive failed attempts of getting the server status before returning with ERROR_RUNTIME.
+
+        Returns: FLAdminAPIResponse
+
+        """
+        failed_attempts = 0
+        start = time.time()
+        while True:
+            try:
+                reply = self.show_stats(TargetType.SERVER)
+                try:
+                    if reply:
+                        met = callback(reply, **kwargs)
+                        if met:
+                            return FLAdminAPIResponse(APIStatus.SUCCESS, {}, None)
+                        fail_attempts = 0
+                    else:
+                        print("Could not get reply from show stats server, trying again later")
+                        failed_attempts += 1
+                except AttributeError:
+                    # if attribute cannot be found, check if app is no longer running to return APIStatus.SUCCESS
+                    if reply.get("details").get("message") == "App is not running":
+                        return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": "Waited until app not running."}, None)
+            except BaseException as e:
+                print("Could not get server stats, trying again later. Exception: ", e)
+                failed_attempts += 1
+
+            now = time.time()
+            if timeout is not None:
+                if now - start >= timeout:
+                    return FLAdminAPIResponse(APIStatus.SUCCESS, {"message": "Waited until timeout."}, None)
+            if failed_attempts > fail_attempts:
+                return FLAdminAPIResponse(
+                    APIStatus.ERROR_RUNTIME,
+                    {
+                        "message": "FL server stats was not obtainable for more than the specified number of "
+                        "fail_attempts. "
+                    },
+                    None,
+                )
+            time.sleep(interval)
diff --git a/nvflare/fuel/hci/client/fl_admin_api_constants.py b/nvflare/fuel/hci/client/fl_admin_api_constants.py
new file mode 100644
index 0000000000..afb5a45993
--- /dev/null
+++ b/nvflare/fuel/hci/client/fl_admin_api_constants.py
@@ -0,0 +1,32 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from enum import Enum
+
+
+class FLDetailKey(str, Enum):
+    """
+    Constants for FL details that can be returned in the FLAdminAPI.
+    """
+
+    RUN_NUMBER = "run_number"
+    APP_NAME = "app_name"
+    REGISTERED_CLIENTS = "registered_clients"
+    CONNECTED_CLIENTS = "connected_clients"
+    SERVER_ENGINE_STATUS = "server_engine_status"
+    SERVER_LOG = "server_log"
+    CLIENT_LOG = "client_log"
+    STATUS_TABLE = "status_table"
+    RESPONSES = "responses"
+    SUBMITTED_MODELS = "submitted_models"
diff --git a/nvflare/fuel/hci/client/fl_admin_api_runner.py b/nvflare/fuel/hci/client/fl_admin_api_runner.py
new file mode 100644
index 0000000000..b85df886e8
--- /dev/null
+++ b/nvflare/fuel/hci/client/fl_admin_api_runner.py
@@ -0,0 +1,219 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+
+from nvflare.fuel.hci.client.fl_admin_api import FLAdminAPI
+from nvflare.fuel.hci.client.fl_admin_api_constants import FLDetailKey
+from nvflare.fuel.hci.client.fl_admin_api_spec import TargetType
+
+
+def api_command_wrapper(api_command_result):
+    """Prints the result of the command and raises RuntimeError to interrupt command sequence if there is an error.
+
+    Args:
+        api_command_result: result of the api command
+
+    """
+    print(api_command_result)
+    if not api_command_result["status"] == "SUCCESS":
+        raise RuntimeError("command was not successful!")
+
+    return api_command_result
+
+
+def wait_until_clients_greater_than_cb(reply, min_clients=2):
+    """Callback in wait_until_server_status for waiting until a minimum number of clients are registered on server.
+
+    Args:
+        reply: reply from checking the server status
+        min_clients: minimum number of clients (default: 2)
+
+    Returns: True if the minimum number of clients has been registered, otherwise False.
+
+    """
+    if reply["details"][FLDetailKey.REGISTERED_CLIENTS] >= min_clients:
+        return True
+    else:
+        return False
+
+
+class FLAdminAPIRunner:
+    def __init__(
+        self,
+        host,
+        port,
+        username,
+        admin_dir,
+        poc=False,
+        debug=False,
+    ):
+        """Initializes and logs into an FLAdminAPI instance. The default locations for certs, keys, and directories
+        are used.
+
+        Args:
+            host: string for
+            port: int for the port number to communicate with server admin
+            username: string of username to login with
+            admin_dir: string of root admin dir containing the startup dir
+            poc: whether to run in poc mode without SSL certs
+            debug: whether to turn on debug mode
+        """
+
+        assert isinstance(host, str), "host must be str"
+        assert isinstance(port, int), "port must be int"
+        assert isinstance(username, str), "username must be str"
+        self.username = username
+        assert isinstance(admin_dir, str), "admin_dir must be str"
+        if poc:
+            self.poc = True
+        else:
+            self.poc = False
+        if debug:
+            debug = True
+
+        ca_cert = os.path.join(admin_dir, "startup", "rootCA.pem")
+        client_cert = os.path.join(admin_dir, "startup", "client.crt")
+        client_key = os.path.join(admin_dir, "startup", "client.key")
+        upload_dir = os.path.join(admin_dir, "transfer")
+        download_dir = os.path.join(admin_dir, "download")
+        if not os.path.isdir(download_dir):
+            os.makedirs(download_dir)
+
+        assert os.path.isdir(admin_dir), f"admin directory does not exist at {admin_dir}"
+        if not self.poc:
+            assert os.path.isfile(ca_cert), f"rootCA.pem does not exist at {ca_cert}"
+            assert os.path.isfile(client_cert), f"client.crt does not exist at {client_cert}"
+            assert os.path.isfile(client_key), f"client.key does not exist at {client_key}"
+
+        # Connect with admin client
+        self.api = FLAdminAPI(
+            host=host,
+            port=port,
+            ca_cert=ca_cert,
+            client_cert=client_cert,
+            client_key=client_key,
+            upload_dir=upload_dir,
+            download_dir=download_dir,
+            poc=self.poc,
+            debug=debug,
+        )
+        if self.poc:
+            reply = self.api.login_with_password("admin", "admin")
+        else:
+            reply = self.api.login(username=self.username)
+        for k in reply.keys():
+            assert "error" not in reply[k].lower(), f"Login not successful with {reply}"
+
+    def run(
+        self,
+        run_number,
+        app,
+        restart_all_first=False,
+        min_clients=2,
+        timeout=2000,
+        shutdown_on_error=False,
+        shutdown_at_end=False,
+    ):
+        """An example script to upload, deploy, and start a specified app(app folder must be in upload_dir already).
+        Prints the command to be executed first so it is easy to follow along as the commands run.
+
+        Args:
+            run_number: run number to use
+            app: app to upload, deploy, and start
+            min_clients: minimum number of clients to have registered on the server after restart all before proceeding (default: 2)
+            timeout: number of seconds to wait in the wait_until_client_status command before the app is aborted in this example (default: 2000)
+            restart_all_first: whether to restart all before the run in order to have clean environment
+            shutdown_on_error: whether to shut down all if there is an error on app startup
+            shutdown_at_end: whether to shut down all at the end of script
+
+        """
+        try:
+            if restart_all_first:
+                print(f'api.restart("{TargetType.ALL}")')
+                api_command_wrapper(self.api.restart(TargetType.ALL))
+                self.attempt_login_until_success()
+                print("api.wait_until_server_status(callback=wait_until_clients_greater_than_cb)")
+                api_command_wrapper(
+                    self.api.wait_until_server_status(
+                        callback=wait_until_clients_greater_than_cb, min_clients=min_clients
+                    )
+                )
+            print("api.check_status(TargetType.SERVER)")
+            api_command_wrapper(self.api.check_status(TargetType.SERVER))
+            print(f"api.set_run_number({run_number})")
+            api_command_wrapper(self.api.set_run_number(run_number))
+            print(f'api.upload_app("{app}")')
+            api_command_wrapper(self.api.upload_app(app))
+            print(f'api.deploy_app("{app}", TargetType.ALL)')
+            api_command_wrapper(self.api.deploy_app(app, TargetType.ALL))
+            print("api.check_status(TargetType.CLIENT)")
+            api_command_wrapper(self.api.check_status(TargetType.CLIENT))
+            print("api.start_app(TargetType.ALL)")
+            api_command_wrapper(self.api.start_app(TargetType.ALL))
+            time.sleep(1)
+            print("api.check_status(TargetType.SERVER)")
+            reply = api_command_wrapper(self.api.check_status(TargetType.SERVER))
+            if shutdown_on_error and reply["details"][FLDetailKey.SERVER_ENGINE_STATUS] == "stopped":
+                print("Server startup failed! Shutdown all...")
+                api_command_wrapper(self.api.shutdown(TargetType.ALL))
+                return
+            # The following wait_until can be put into a loop that has other behavior other than aborting after the
+            # timeout is reached for actual apps. This code is just a demonstration of running an app expected to stop
+            # before the timeout.
+            print("api.wait_until_client_status()")
+            wait_result = api_command_wrapper(self.api.wait_until_client_status(timeout=timeout))
+            if wait_result.get("details"):
+                if wait_result.get("details").get("message"):
+                    if wait_result.get("details").get("message") == "Waited until timeout.":
+                        print(
+                            "aborting because waited until timeout and there are clients that have still not stopped."
+                        )
+                        print("api.abort(TargetType.ALL)")
+                        api_command_wrapper(self.api.abort(TargetType.ALL))
+            print("api.check_status(TargetType.SERVER)")
+            api_command_wrapper(self.api.check_status(TargetType.SERVER))
+            # now server engine status should be stopped
+            time.sleep(10)  # wait for clients to stop in case they take longer than server to stop
+            print("api.check_status(TargetType.CLIENT)")
+            api_command_wrapper(self.api.check_status(TargetType.CLIENT))
+            if shutdown_at_end:
+                print("api.shutdown(TargetType.ALL)")
+                api_command_wrapper(self.api.shutdown(TargetType.ALL))
+        except RuntimeError as e:
+            print(f"There was an exception: {e}")
+            if shutdown_on_error:
+                print("Attempting shutdown all...")
+                try:
+                    api_command_wrapper(self.api.shutdown(TargetType.ALL))
+                except RuntimeError as e:
+                    print(f"There was an exception while attempting shutdown all: {e}")
+
+    def attempt_login_until_success(self):
+        logged_in = False
+        while not logged_in:
+            reply = None
+            try:
+                if self.poc:
+                    reply = self.api.login_with_password("admin", "admin")
+                else:
+                    print("api.login()")
+                    reply = api_command_wrapper(self.api.login(self.username))
+            except RuntimeError:
+                time.sleep(10)
+                pass
+            if reply and reply["status"] == "SUCCESS":
+                logged_in = True
+        return True
diff --git a/nvflare/fuel/hci/client/fl_admin_api_spec.py b/nvflare/fuel/hci/client/fl_admin_api_spec.py
new file mode 100644
index 0000000000..4cf0e8c326
--- /dev/null
+++ b/nvflare/fuel/hci/client/fl_admin_api_spec.py
@@ -0,0 +1,363 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Callable, List, Optional
+
+from nvflare.fuel.hci.client.api_status import APIStatus
+
+
+class FLAdminAPIResponse(dict):
+    """
+    Structure containing the response of calls to the api as key value pairs. The status key is the primary indicator of
+    the success of a call and can contain APIStatus.SUCCESS or another APIStatus. Most calls will return additional
+    information in the details key, which is also a dictionary of key value pairs. The raw key can optionally have
+    the underlying response from AdminAPI when relevant, particularly when data is received from the server and the
+    status of a call is APIStatus.ERROR_RUNTIME to provide additional information.
+
+    Note that the status in this response primarily indicates that the command submitted successfully. Depending on the
+    command and especially for calls to multiple clients, the contents of details or the raw response should be examined
+    to determine if the execution of the command was successful for each specific client.
+    """
+
+    def __init__(self, status: APIStatus, details: dict = None, raw: dict = None):
+        super().__init__()
+        self["status"] = status
+        if details:
+            self["details"] = details
+        if raw:
+            self["raw"] = raw
+
+
+class APISyntaxError(Exception):
+    pass
+
+
+class TargetType(str, Enum):
+    ALL = "all"
+    SERVER = "server"
+    CLIENT = "client"
+
+
+class FLAdminAPISpec(ABC):
+    @abstractmethod
+    def check_status(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        """
+        Checks and returns the FL status. If target_type is server, the call does not wait for the server to retrieve
+        information on the clients but returns the last information the server had at the time this call is made.
+
+        If target_type is client, specific clients can be specified in targets, and this call generally takes longer
+        than the function to just check the FL server status because this one waits for communication from the server to
+        client then back.
+
+        Note that this is still the previous training check_status, and there will be a new call to get status through
+        InfoCollector, which will be able to get information from components.
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def set_run_number(self, run_number: int) -> FLAdminAPIResponse:
+        """
+        Sets a current run number.
+
+        Args:
+            run_number: run number in order to set for the current experiment, must be integer greater than 0
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def delete_run_number(self, run_number: int) -> FLAdminAPIResponse:
+        """
+        Deletes a specified run number. This deletes the run folder corresponding to the run number on the server and
+        all connected clients. This is not reversible.
+
+        Args:
+            run_number: run number for the run folder to delete
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def upload_app(self, app: str) -> FLAdminAPIResponse:
+        """
+        Uploads specified app to the upload directory of FL server. Currently assumes app is in the upload_dir set in
+        API init.
+
+        Args:
+            app: name of the folder in upload_dir to upload
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def deploy_app(self, app: str, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        """
+        Issues a command to deploy the specified app to the specified target for the current run number. The app must
+        be already uploaded and available on the server.
+
+        Args:
+            app: name of app to deploy
+            target_type: server | client | all
+            targets: if target_type is client, targets can optionally be a list of client names
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def start_app(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        """
+        Issue a command to start the deployed app for the current run number at the specified target.
+
+        Args:
+            target_type: server | client | all
+            targets: if target_type is client, targets can optionally be a list of client names
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def abort(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        """Issue a command to abort training.
+
+        Args:
+            target_type: server | client
+            targets: if target_type is client, targets can optionally be a list of client names
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def restart(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        """Issue a command to restart the specified target. If the target is server, all FL clients will be restarted
+        as well.
+
+        Args:
+            target_type: server | client
+            targets: if target_type is client, targets can optionally be a list of client names
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def shutdown(self, target_type: TargetType, targets: Optional[List[str]] = None) -> FLAdminAPIResponse:
+        """Issue a command to stop FL entirely for a specific FL client or specific FL clients. Note that the targets
+        will not be able to start with an API command after shutting down.
+
+        Args:
+            target_type: server | client
+            targets: if target_type is client, targets can optionally be a list of client names
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def remove_client(self, targets: List[str]) -> FLAdminAPIResponse:
+        """Issue a command to remove a specific FL client or FL clients. Note that the targets
+        will not be able to start with an API command after shutting down.
+
+        Args:
+            targets: a list of client names
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def set_timeout(self, timeout: float) -> FLAdminAPIResponse:
+        """
+        Sets the timeout for admin commands on the server.
+
+        Args:
+            timeout: timeout of admin commands to set on the server
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def get_available_apps_to_upload(self):
+        pass
+
+    @abstractmethod
+    def ls_target(self, target: str, options: str = None, path: str = None) -> FLAdminAPIResponse:
+        """
+        Sends the shell command to get the directory listing of the target allowing for options that the ls command
+        of admin client allows.
+
+        Args:
+            target:  either server or single client's client name.
+            options: the options string as provided to the ls command for admin client.
+            path:    optionally, the path to specify
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def cat_target(self, target: str, options: str = None, file: str = None) -> FLAdminAPIResponse:
+        """
+        Sends the shell command to get the contents of the target's specified file allowing for options that the cat
+        command of admin client allows.
+
+        Args:
+            target:  either server or single client's client name.
+            options: the options string as provided to the ls command for admin client.
+            file:    the path to the file to return the contents of
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def tail_target_log(self, target: str, options: str = None) -> FLAdminAPIResponse:
+        """Returns the end of target's log allowing for options that the tail of admin client allows.
+
+        Args:
+            target:  either server or single client's client name.
+            options: the options string as provided to the tail command for admin client. For this command, "-n" can be
+                     used to specify the number of lines for example "-n 100", or "-c" can specify the number of bytes.
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def env_target(self, target: str) -> FLAdminAPIResponse:
+        """Get the environment variables of the specified target.
+
+        Args:
+            target:  either server or single client's client name.
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def get_working_directory(self, target: str) -> FLAdminAPIResponse:
+        """Gets the workspace root directory of the specified target.
+
+        Args:
+            target:  either server or single client's client name.
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def grep_target(
+        self, target: str, options: str = None, pattern: str = None, file: str = None
+    ) -> FLAdminAPIResponse:
+        """
+        Sends the shell command to grep the contents of the target's specified file allowing for options that the grep
+        command of admin client allows.
+
+        Args:
+            target:  either server or single client's client name.
+            options: the options string as provided to the grep command for admin client.
+            pattern: the pattern to search for
+            file:    the path to the file to grep
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def show_stats(self) -> FLAdminAPIResponse:
+        """Gets and shows stats from the Info Collector.
+
+        Returns: FLAdminAPIResponse
+
+        """
+
+    @abstractmethod
+    def show_errors(self) -> FLAdminAPIResponse:
+        """Gets and shows errors from the Info Collector.
+
+        Returns: FLAdminAPIResponse
+
+        """
+
+    @abstractmethod
+    def reset_errors(self) -> FLAdminAPIResponse:
+        """Resets the collector errors.
+
+        Returns: FLAdminAPIResponse
+
+        """
+
+    @abstractmethod
+    def get_connected_client_list(self) -> FLAdminAPIResponse:
+        """A convenience function to get a list of the clients currently connected to the FL server through the check
+        status server call. Note that this returns the client list based on the last known statuses on the server, so it
+        can be possible for a client to be disconnected and not yet removed from the list of connected clients.
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
+
+    @abstractmethod
+    def wait_until_server_status(
+        self,
+        interval: int = 20,
+        timeout: int = None,
+        callback: Callable[[FLAdminAPIResponse], bool] = None,
+        fail_attempts: int = 3,
+    ) -> FLAdminAPIResponse:
+        """Wait until provided callback returns True, with the option to specify a timeout and interval to
+        check the server status. If no callback function is provided, the default callback returns True when the server
+        status is "training stopped". A custom callback can be provided to add logic to handle checking for other
+        conditions. A timeout should be set in case there are any error conditions that result in the system being stuck
+        in a state where the callback never returns True.
+
+        Args:
+            interval: in seconds, the time between consecutive checks of the server
+            timeout: if set, the amount of time this function will run until before returning a response message
+            callback: the reply from check_status_server() will be passed to the callback, along with any additional kwargs
+            which can go on to perform additional logic.
+            fail_attempts: number of consecutive failed attempts of getting the server status before returning with ERROR_RUNTIME.
+
+        Returns: FLAdminAPIResponse
+
+        """
+        pass
diff --git a/nvflare/fuel/hci/client/lib.py b/nvflare/fuel/hci/client/lib.py
new file mode 100644
index 0000000000..5a78440b08
--- /dev/null
+++ b/nvflare/fuel/hci/client/lib.py
@@ -0,0 +1,636 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import cmd
+import getpass
+import json
+import os
+import socket
+import ssl
+import time
+import traceback
+from datetime import datetime
+
+from nvflare.fuel.hci.cmd_arg_utils import join_args, split_to_args
+from nvflare.fuel.hci.conn import Connection, receive_and_process
+from nvflare.fuel.hci.proto import make_error
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandRegister, CommandSpec
+from nvflare.fuel.hci.security import get_certificate_common_name, hash_password, verify_password
+from nvflare.fuel.hci.table import Table
+
+
+class _BuiltInCmdModule(CommandModule):
+    def get_spec(self):
+        return CommandModuleSpec(
+            name="",
+            cmd_specs=[
+                CommandSpec(name="bye", description="exit from the client", usage="bye", handler_func=None),
+                CommandSpec(name="help", description="get command help information", usage="help", handler_func=None),
+                CommandSpec(
+                    name="lpwd", description="print local work dir of the admin client", usage="lpwd", handler_func=None
+                ),
+            ],
+        )
+
+
+class ReplyProcessor(object):
+    def reply_start(self, client, reply_json):
+        pass
+
+    def process_string(self, client, item: str):
+        pass
+
+    def process_success(self, client, item: str):
+        pass
+
+    def process_error(self, client, err: str):
+        pass
+
+    def process_table(self, client, table: Table):
+        pass
+
+    def process_dict(self, client, data: dict):
+        pass
+
+    def process_shutdown(self, client, msg: str):
+        pass
+
+    def process_token(self, client, token: str):
+        pass
+
+    def protocol_error(self, client, err: str):
+        pass
+
+    def reply_done(self, client):
+        pass
+
+
+class _RegularReplyProcessor(ReplyProcessor):
+    def process_shutdown(self, client, msg: str):
+        client.shutdown_received = True
+        client.shutdown_msg = msg
+
+    def process_string(self, client, item: str):
+        client.write_string(item)
+
+    def process_error(self, client, err: str):
+        client.write_error(err)
+
+    def process_table(self, client, table: Table):
+        client.write_table(table)
+
+    def process_dict(self, client, data: dict):
+        client.write_dict(data)
+
+    def protocol_error(self, client, err: str):
+        client.write_error(err)
+
+    def reply_done(self, client):
+        client.flush()
+
+
+class _LoginReplyProcessor(ReplyProcessor):
+    def reply_start(self, client, reply_json):
+        client.login_result = "error"
+
+    def process_string(self, client, item: str):
+        client.login_result = item
+
+    def process_token(self, client, token: str):
+        client.token = token
+
+
+class _CmdListReplyProcessor(ReplyProcessor):
+    def reply_start(self, client, reply_json):
+        client.server_cmd_received = False
+
+    def process_table(self, client, table: Table):
+        for i in range(len(table.rows)):
+            if i == 0:
+                # this is header
+                continue
+
+            row = table.rows[i]
+            if len(row) < 5:
+                return
+
+            scope = row[0]
+            cmd_name = row[1]
+            desc = row[2]
+            usage = row[3]
+            confirm = row[4]
+
+            if confirm == "auth" and not client.require_login:
+                # the user is not authenticated - skip this command
+                continue
+
+            client.server_cmd_reg.add_command(
+                scope_name=scope,
+                cmd_name=cmd_name,
+                desc=desc,
+                usage=usage,
+                handler=None,
+                authz_func=None,
+                visible=True,
+                confirm=confirm,
+            )
+
+        client.server_cmd_received = True
+        client.server_cmd_reg.finalize(client.register_command)
+
+
+class AdminClient(cmd.Cmd):
+    """Admin command prompt for submitting admin commands to the server through the CLI.
+
+    Args:
+        host: cn provisioned for the project, with this fully qualified domain name resolving to the IP of the FL server
+        port: port provisioned as admin_port for FL admin communication, by default provisioned as 8003, must be int
+        prompt: prompt to use for the command prompt
+        ca_cert: path to CA Cert file, by default provisioned rootCA.pem
+        client_cert: path to admin client Cert file, by default provisioned as client.crt
+        client_key: path to admin client Key file, by default provisioned as client.key
+        server_cn: server cn
+        require_login: whether or not to require login
+        credential_type: what type of credential to use
+        cmd_modules: command modules to load and register
+        debug: Whether or not to print debug messages. False by default.
+    """
+
+    def __init__(
+        self,
+        host,
+        port,
+        prompt="> ",
+        ca_cert=None,
+        client_cert=None,
+        client_key=None,
+        server_cn=None,
+        require_login=False,
+        credential_type="password",
+        cmd_modules=None,
+        debug=False,
+    ):
+        cmd.Cmd.__init__(self)
+        self.intro = "Type help or ? to list commands.\n"
+        self.prompt = prompt
+        self.host = host
+        self.port = port
+        self.ca_cert = ca_cert
+        self.client_cert = client_cert
+        self.client_key = client_key
+        self.server_cn = server_cn
+        self.require_login = require_login
+        self.credential_type = credential_type
+        self.user_name = None
+        self.pwd = None
+        self.token = None
+        self.login_result = None
+        self.server_cmd_reg = CommandRegister(app_ctx=self)
+        self.client_cmd_reg = CommandRegister(app_ctx=self)
+        self.server_cmd_received = False
+        self.debug = debug
+        self.shutdown_received = False
+        self.shutdown_msg = None
+        self.reply_processor = None
+        self.all_cmds = []
+        self.out_file = None
+        self.no_stdout = False
+
+        assert credential_type in ["password", "cert"], "invalid credential_type {}".format(credential_type)
+
+        self.client_cmd_reg.register_module(_BuiltInCmdModule())
+        if cmd_modules:
+            assert isinstance(cmd_modules, list), "cmd_modules must be a list"
+            for m in cmd_modules:
+                assert isinstance(m, CommandModule), "cmd_modules must be a list of CommandModule"
+                self.client_cmd_reg.register_module(m, include_invisible=False)
+
+        self.client_cmd_reg.finalize(self.register_command)
+
+    def _set_output_file(self, file, no_stdout):
+        self._close_output_file()
+        self.out_file = file
+        self.no_stdout = no_stdout
+
+    def _close_output_file(self):
+        if self.out_file:
+            self.out_file.close()
+            self.out_file = None
+        self.no_stdout = False
+
+    def register_command(self, cmd_entry):
+        self.all_cmds.append(cmd_entry.name)
+
+    def do_bye(self, arg):
+        "exit from the client"
+        if self.require_login:
+            self.server_execute("_logout")
+        return True
+
+    def do_lpwd(self, arg):
+        "print local current work dir"
+        self.write_string(os.getcwd())
+
+    def emptyline(self):
+        return
+
+    def _show_one_command(self, cmd_name, reg):
+        entries = reg.get_command_entries(cmd_name)
+        if len(entries) <= 0:
+            self.write_string("Undefined command {}\n".format(cmd_name))
+            return
+
+        for e in entries:
+            if not e.visible:
+                continue
+
+            if len(e.scope.name) > 0:
+                self.write_string("Command: {}.{}".format(e.scope.name, cmd_name))
+            else:
+                self.write_string("Command: {}".format(cmd_name))
+
+            self.write_string("Description: {}".format(e.desc))
+            self.write_string("Usage: {}\n".format(e.usage))
+
+    def _show_commands(self, reg: CommandRegister):
+        table = Table(["Scope", "Command", "Description"])
+        for scope_name in sorted(reg.scopes):
+            scope = reg.scopes[scope_name]
+            for cmd_name in sorted(scope.entries):
+                e = scope.entries[cmd_name]
+                if e.visible:
+                    table.add_row([scope_name, cmd_name, e.desc])
+        self.write_table(table)
+
+    def do_help(self, arg):
+        if len(arg) <= 0:
+            self.write_string("Client Commands")
+            self._show_commands(self.client_cmd_reg)
+
+            self.write_string("\nServer Commands")
+            self._show_commands(self.server_cmd_reg)
+        else:
+            server_cmds = []
+            local_cmds = []
+            parts = arg.split()
+            for p in parts:
+                entries = self.client_cmd_reg.get_command_entries(p)
+                if len(entries) > 0:
+                    local_cmds.append(p)
+
+                entries = self.server_cmd_reg.get_command_entries(p)
+                if len(entries) > 0:
+                    server_cmds.append(p)
+
+            if len(local_cmds) > 0:
+                self.write_string("Client Commands")
+                self.write_string("---------------")
+                for cmd_name in local_cmds:
+                    self._show_one_command(cmd_name, self.client_cmd_reg)
+
+            if len(server_cmds) > 0:
+                self.write_string("Server Commands")
+                self.write_string("---------------")
+                for cmd_name in server_cmds:
+                    self._show_one_command(cmd_name, self.server_cmd_reg)
+
+    def complete(self, text, state):
+        results = [x + " " for x in self.all_cmds if x.startswith(text)] + [None]
+        return results[state]
+
+    def default(self, line):
+        self._close_output_file()
+        try:
+            return self._do_default(line)
+        except KeyboardInterrupt:
+            self.write_stdout("\n")
+        except BaseException as ex:
+            if self.debug:
+                traceback.print_exc()
+            self.write_stdout("exception occurred: {}".format(ex))
+        self._close_output_file()
+
+    def _do_default(self, line):
+        args = split_to_args(line)
+        cmd_name = args[0]
+
+        # check for file output
+        out_file_name = None
+        no_stdout = False
+        out_arg_idx = 0
+        for i in range(len(args)):
+            arg = args[i]
+            if arg.startswith(">") and out_file_name is not None:
+                self.write_error("only one output file is supported")
+                return
+
+            if arg.startswith(">>"):
+                # only output to file
+                out_file_name = arg[2:]
+                no_stdout = True
+                out_arg_idx = i
+            elif arg.startswith(">"):
+                # only output to file
+                out_file_name = arg[1:]
+                no_stdout = False
+                out_arg_idx = i
+
+        if out_file_name is not None:
+            if len(out_file_name) <= 0:
+                self.write_error("output file name must not be empty")
+                return
+            args.pop(out_arg_idx)
+            line = join_args(args)
+            try:
+                out_file = open(out_file_name, "w")
+            except BaseException as ex:
+                self.write_error("cannot open file {}: {}".format(out_file_name, ex))
+                return
+
+            self._set_output_file(out_file, no_stdout)
+
+        # check client command first
+        entries = self.client_cmd_reg.get_command_entries(cmd_name)
+        if len(entries) > 1:
+            self.write_string("Ambiguous client command {} - qualify with scope".format(cmd_name))
+            return
+        elif len(entries) == 1:
+            ent = entries[0]
+            return ent.handler(args, self.client_cmd_reg.app_ctx)
+
+        entries = self.server_cmd_reg.get_command_entries(cmd_name)
+        if len(entries) <= 0:
+            self.write_string("Undefined server command {}".format(cmd_name))
+            return
+        elif len(entries) > 1:
+            self.write_string("Ambiguous server command {} - qualify with scope".format(cmd_name))
+            return
+
+        ent = entries[0]
+        confirm_method = ent.confirm
+        if ent.confirm == "auth":
+            if self.credential_type == "password":
+                confirm_method = "pwd"
+            elif self.user_name:
+                confirm_method = "username"
+            else:
+                confirm_method = "yesno"
+
+        if confirm_method == "yesno":
+            answer = input("Are you sure (Y/N): ")
+            answer = answer.lower()
+            if answer != "y" and answer != "yes":
+                return
+        elif confirm_method == "username":
+            answer = input("Confirm with User Name: ")
+            if answer != self.user_name:
+                self.write_string("user name mismatch")
+                return
+        elif confirm_method == "pwd":
+            pwd = getpass.getpass("Enter password to confirm: ")
+            if not verify_password(self.pwd, pwd):
+                self.write_string("Not authenticated")
+                return
+
+        self.server_execute(line)
+        if self.shutdown_received:
+            # exit the client
+            self.write_string(self.shutdown_msg)
+            return True
+
+    def _send_to_sock(self, sock, command, process_json_func):
+        conn = Connection(sock, self)
+        conn.append_command(command)
+        if self.token:
+            conn.append_token(self.token)
+
+        conn.close()
+        ok = receive_and_process(sock, process_json_func)
+        if not ok:
+            process_json_func(
+                make_error("Failed to communicate with Admin Server {} on {}".format(self.host, self.port))
+            )
+
+    def _process_server_reply(self, resp):
+        if self.debug:
+            print("Server Reply: {}".format(resp))
+
+        reply_processor = self.reply_processor
+        if reply_processor is None:
+            reply_processor = _RegularReplyProcessor()
+
+        reply_processor.reply_start(self, resp)
+
+        if resp is not None:
+            data = resp["data"]
+            for item in data:
+                it = item["type"]
+                if it == "string":
+                    reply_processor.process_string(self, item["data"])
+                elif it == "success":
+                    reply_processor.process_success(self, item["data"])
+                elif it == "error":
+                    reply_processor.process_error(self, item["data"])
+                elif it == "table":
+                    table = Table(None)
+                    table.set_rows(item["rows"])
+                    reply_processor.process_table(self, table)
+                elif it == "dict":
+                    reply_processor.process_dict(self, item["data"])
+                elif it == "token":
+                    reply_processor.process_token(self, item["data"])
+                elif it == "shutdown":
+                    reply_processor.process_shutdown(self, item["data"])
+                else:
+                    reply_processor.protocol_error(self, "Invalid item type: " + it)
+        else:
+            reply_processor.protocol_error(self, "Protocol Error")
+
+        reply_processor.reply_done(self)
+
+    def _try_command(self, command, reply_processor):
+        self.reply_processor = reply_processor
+        process_json_func = self._process_server_reply
+        try:
+            if self.ca_cert and self.client_cert:
+                # SSL communication
+                ctx = ssl.create_default_context()
+                ctx.verify_mode = ssl.CERT_REQUIRED
+                ctx.check_hostname = False
+
+                ctx.load_verify_locations(self.ca_cert)
+                ctx.load_cert_chain(certfile=self.client_cert, keyfile=self.client_key)
+
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+                    with ctx.wrap_socket(sock) as ssock:
+                        ssock.connect((self.host, self.port))
+                        if self.server_cn:
+                            # validate server CN
+                            cn = get_certificate_common_name(ssock.getpeercert())
+                            if cn != self.server_cn:
+                                process_json_func(
+                                    make_error("wrong server: expecting {} but connected {}".format(self.server_cn, cn))
+                                )
+                                return
+
+                        self._send_to_sock(ssock, command, process_json_func)
+            else:
+                # Non-SSL
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+                    sock.connect((self.host, self.port))
+                    self._send_to_sock(sock, command, process_json_func)
+        except Exception as ex:
+            if self.debug:
+                traceback.print_exc()
+
+            process_json_func(
+                make_error("Failed to communicate with Admin Server {} on {}: {}".format(self.host, self.port, ex))
+            )
+
+    def server_execute(self, command, reply_processor=None):
+        start = time.time()
+        self._try_command(command, reply_processor)
+        secs = time.time() - start
+        usecs = int(secs * 1000000)
+        done = "Done [{} usecs] {}".format(usecs, datetime.now())
+        self.write_stdout(done)
+
+    def cmdloop(self, intro=None):
+        """Repeatedly issue a prompt, accept input, parse an initial prefix
+        off the received input, and dispatch to action methods, passing them
+        the remainder of the line as argument.
+
+        Overriding what is in cmd.Cmd to handle exiting client on Ctrl+D (EOF).
+        """
+
+        self.preloop()
+        if self.use_rawinput and self.completekey:
+            try:
+                import readline
+
+                self.old_completer = readline.get_completer()
+                readline.set_completer(self.complete)
+                readline.parse_and_bind(self.completekey + ": complete")
+            except ImportError:
+                pass
+        try:
+            if intro is not None:
+                self.intro = intro
+            if self.intro:
+                self.stdout.write(str(self.intro) + "\n")
+            stop = None
+            while not stop:
+                if self.cmdqueue:
+                    line = self.cmdqueue.pop(0)
+                else:
+                    if self.use_rawinput:
+                        try:
+                            line = input(self.prompt)
+                        except EOFError:
+                            line = "bye"
+                        except KeyboardInterrupt:
+                            self.stdout.write("\n")
+                            line = "\n"
+                    else:
+                        self.stdout.write(self.prompt)
+                        self.stdout.flush()
+                        line = self.stdin.readline()
+                        if not len(line):
+                            line = "EOF"
+                        else:
+                            line = line.rstrip("\r\n")
+                line = self.precmd(line)
+                stop = self.onecmd(line)
+                stop = self.postcmd(stop, line)
+            self.postloop()
+        finally:
+            if self.use_rawinput and self.completekey:
+                try:
+                    import readline
+
+                    readline.set_completer(self.old_completer)
+                except ImportError:
+                    pass
+
+    def run(self):
+        if self.require_login:
+            user_name = input("User Name: ")
+
+            if self.credential_type == "password":
+                while True:
+                    pwd = getpass.getpass("Password: ")
+                    self._try_command("_login {} {}".format(user_name, pwd), _LoginReplyProcessor())
+                    if self.login_result == "OK":
+                        self.user_name = user_name
+                        self.pwd = hash_password(pwd)
+                        break
+                    elif self.login_result == "REJECT":
+                        print("Incorrect password - please try again")
+                    else:
+                        print("Communication Error - please try later")
+                        return
+            elif self.credential_type == "cert":
+                self._try_command("_cert_login {}".format(user_name), _LoginReplyProcessor())
+                if self.login_result == "OK":
+                    self.user_name = user_name
+                elif self.login_result == "REJECT":
+                    print("Incorrect user name or certificate")
+                    return
+                else:
+                    print("Communication Error - please try later")
+                    return
+
+        # get command list from server
+        self._try_command("_commands", _CmdListReplyProcessor())
+        if not self.server_cmd_received:
+            print("Failed to handshake with server - please try later")
+            return
+
+        self.cmdloop(intro='Type ? to list commands; type "? cmdName" to show usage of a command.')
+
+    def write_stdout(self, data: str):
+        self.stdout.write(data + "\n")
+
+    def write_string(self, data: str):
+        content = data + "\n"
+        if not self.no_stdout:
+            self.stdout.write(content)
+        if self.out_file:
+            self.out_file.write(content)
+
+    def write_table(self, table: Table):
+        if not self.no_stdout:
+            table.write(self.stdout)
+        if self.out_file:
+            table.write(self.out_file)
+
+    def write_dict(self, data: dict):
+        content = json.dumps(data, indent=2) + "\n"
+        if not self.no_stdout:
+            self.stdout.write(content)
+        if self.out_file:
+            self.out_file.write(content)
+
+    def write_error(self, err: str):
+        content = "Error: " + err + "\n"
+        if not self.no_stdout:
+            self.stdout.write(content)
+        if self.out_file:
+            self.out_file.write(content)
+
+    def flush(self):
+        if not self.no_stdout:
+            self.stdout.flush()
+        if self.out_file:
+            self.out_file.flush()
diff --git a/nvflare/fuel/hci/cmd_arg_utils.py b/nvflare/fuel/hci/cmd_arg_utils.py
new file mode 100644
index 0000000000..765ab01944
--- /dev/null
+++ b/nvflare/fuel/hci/cmd_arg_utils.py
@@ -0,0 +1,69 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import io
+import re
+import shlex
+from typing import List
+
+
+def split_to_args(line: str) -> List[str]:
+    if '"' in line:
+        return shlex.split(line)
+    else:
+        line = re.sub(" +", " ", line)
+        return line.split(" ")
+
+
+def join_args(segs: List[str]) -> str:
+    result = ""
+    sep = ""
+    for a in segs:
+        parts = a.split()
+        if len(parts) < 2:
+            p = parts[0]
+        else:
+            p = '"' + a + '"'
+        result = result + sep + p
+        sep = " "
+
+    return result
+
+
+class ArgValidator(argparse.ArgumentParser):
+    """
+    Validator for admin shell commands that uses argparse to check arguments and get usage through print_help.
+    """
+
+    def __init__(self, name):
+        argparse.ArgumentParser.__init__(self, prog=name, add_help=False)
+        self.err = ""
+
+    def error(self, message):
+        self.err = message
+
+    def validate(self, args):
+        try:
+            result = self.parse_args(args)
+            return self.err, result
+        except BaseException:
+            return 'argument error; try "? cmdName to show supported usage for a command"', None
+
+    def get_usage(self) -> str:
+        buffer = io.StringIO()
+        self.print_help(buffer)
+        usage_output = buffer.getvalue().split("\n", 1)[1]
+        buffer.close()
+        return usage_output
diff --git a/nvflare/fuel/hci/conn.py b/nvflare/fuel/hci/conn.py
new file mode 100644
index 0000000000..3ed7e95978
--- /dev/null
+++ b/nvflare/fuel/hci/conn.py
@@ -0,0 +1,193 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from nvflare.fuel.common.ctx import BaseContext
+
+from .proto import Buffer, validate_proto
+from .table import Table
+
+# ASCII Message Format:
+#
+# Only ASCII chars can be used in message;
+# A message consists of multiple lines, each ended with the LINE_END char;
+# The message is ended with the ALL_END char.
+
+LINE_END = "\x03"  # Indicates the end of the line (end of text)
+ALL_END = "\x04"  # Marks the end of a complete transmission (End of Transmission)
+
+
+MAX_MSG_SIZE = 1024
+
+
+def receive_til_end(sock, end=ALL_END):
+    total_data = []
+
+    while True:
+        data = str(sock.recv(1024), "utf-8")
+        if end in data:
+            total_data.append(data[: data.find(end)])
+            break
+
+        total_data.append(data)
+
+    result = "".join(total_data)
+    return result.replace(LINE_END, "")
+
+
+# Returns:
+# seg1 - the text before the end
+# seg2 - the text after the end
+# if end is not found, seg2 is None
+# if end is found, seg2 is a string
+def _split_data(data: str):
+    # first determine whether the data contains ALL_END
+    # anything after ALL_END is dropped
+    all_done = False
+    idx = data.find(ALL_END)
+    if idx >= 0:
+        data = data[:idx]
+        all_done = True
+
+    # find lines separated by LINE_END
+    parts = data.split(LINE_END)
+    return parts, all_done
+
+
+def _process_one_line(line: str, process_json_func):
+    """
+    Validate and process one line, which should be a str containing a JSON document.
+    """
+    json_data = validate_proto(line)
+    process_json_func(json_data)
+
+
+def receive_and_process(sock, process_json_func):
+    """Receives and sends lines to process with process_json_func."""
+    leftover = ""
+    while True:
+        data = str(sock.recv(MAX_MSG_SIZE), "utf-8")
+        if len(data) <= 0:
+            return False
+
+        segs, all_done = _split_data(data)
+        if all_done:
+            for seg in segs:
+                line = leftover + seg
+                if len(line) > 0:
+                    _process_one_line(line, process_json_func)
+                leftover = ""
+            return True
+
+        for i in range(len(segs) - 1):
+            line = leftover + segs[i]
+            if len(line) > 0:
+                _process_one_line(line, process_json_func)
+            leftover = ""
+
+        leftover += segs[len(segs) - 1]
+
+
+class Connection(BaseContext):
+    """
+    Object containing connection information and buffer to build and send a line with socket passed in at init.
+    """
+
+    def __init__(self, sock, server):
+        BaseContext.__init__(self)
+        self.sock = sock
+        self.server = server
+        self.app_ctx = None
+        self.ended = False
+        self.request = None
+        self.command = None
+        self.args = None
+        self.buffer = Buffer()
+
+    def _send_line(self, line: str, all_end=False):
+        """
+        If not self.ended, send line with sock.
+        """
+        if self.ended:
+            return
+
+        if all_end:
+            end = ALL_END
+            self.ended = True
+        else:
+            end = LINE_END
+
+        self.sock.sendall(bytes(line + end, "utf-8"))
+
+    def append_table(self, headers: List[str]) -> Table:
+        return self.buffer.append_table(headers)
+
+    def append_string(self, data: str, flush=False):
+        self.buffer.append_string(data)
+        if flush:
+            self.flush()
+
+    def append_success(self, data: str, flush=False):
+        self.buffer.append_success(data)
+        if flush:
+            self.flush()
+
+    def append_dict(self, data: dict, flush=False):
+        self.buffer.append_dict(data)
+        if flush:
+            self.flush()
+
+    def append_error(self, data: str, flush=False):
+        self.buffer.append_error(data)
+        if flush:
+            self.flush()
+
+    def append_command(self, cmd: str, flush=False):
+        self.buffer.append_command(cmd)
+        if flush:
+            self.flush()
+
+    def append_token(self, token: str, flush=False):
+        self.buffer.append_token(token)
+        if flush:
+            self.flush()
+
+    def append_shutdown(self, msg: str, flush=False):
+        self.buffer.append_shutdown(msg)
+        if flush:
+            self.flush()
+
+    def append_any(self, data, flush=False):
+        if data is None:
+            return
+
+        if isinstance(data, str):
+            self.append_string(data, flush)
+        elif isinstance(data, dict):
+            self.append_dict(data, flush)
+        else:
+            self.append_error("unsupported data type {}".format(type(data)))
+
+    def flush(self):
+        line = self.buffer.encode()
+        if line is None or len(line) <= 0:
+            return
+
+        self.buffer.reset()
+        self._send_line(line, all_end=False)
+
+    def close(self):
+        self.flush()
+        self._send_line("", all_end=True)
diff --git a/nvflare/fuel/hci/file_transfer_defs.py b/nvflare/fuel/hci/file_transfer_defs.py
new file mode 100644
index 0000000000..55b896343f
--- /dev/null
+++ b/nvflare/fuel/hci/file_transfer_defs.py
@@ -0,0 +1,26 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Constants for file transfer command module.
+"""
+
+SERVER_MODULE_NAME = "file_transfer"
+SERVER_CMD_UPLOAD_TEXT = "_upload_text_file"
+SERVER_CMD_DOWNLOAD_TEXT = "_download_text_file"
+SERVER_CMD_UPLOAD_BINARY = "_upload_binary_file"
+SERVER_CMD_DOWNLOAD_BINARY = "_download_binary_file"
+SERVER_CMD_UPLOAD_FOLDER = "_upload_folder"
+SERVER_CMD_DOWNLOAD_FOLDER = "_download_folder"
+SERVER_CMD_INFO = "_info"
diff --git a/nvflare/fuel/hci/proto.py b/nvflare/fuel/hci/proto.py
new file mode 100644
index 0000000000..36cf454651
--- /dev/null
+++ b/nvflare/fuel/hci/proto.py
@@ -0,0 +1,107 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from datetime import datetime
+from typing import List
+
+from .table import Table
+
+
+class Buffer(object):
+    """
+    Buffer to append to for :class:`nvflare.fuel.hci.conn.Connection`.
+    """
+
+    def __init__(self):
+        self.output = {"time": "{}".format(datetime.now()), "data": []}
+
+    def append_table(self, headers: List[str]) -> Table:
+        t = Table(headers)
+        self.output["data"].append({"type": "table", "rows": t.rows})
+        return t
+
+    def append_string(self, data: str):
+        self.output["data"].append({"type": "string", "data": data})
+
+    def append_dict(self, data: dict):
+        self.output["data"].append({"type": "dict", "data": data})
+
+    def append_success(self, data: str):
+        self.output["data"].append({"type": "success", "data": data})
+
+    def append_error(self, data: str):
+        self.output["data"].append({"type": "error", "data": data})
+
+    def append_command(self, cmd: str):
+        self.output["data"].append({"type": "command", "data": cmd})
+
+    def append_token(self, token: str):
+        self.output["data"].append({"type": "token", "data": token})
+
+    def append_shutdown(self, msg: str):
+        self.output["data"].append({"type": "shutdown", "data": msg})
+
+    def encode(self):
+        if len(self.output["data"]) <= 0:
+            return None
+
+        return json.dumps(self.output)
+
+    def reset(self):
+        self.output = {"time": "{}".format(datetime.now()), "data": []}
+
+
+def make_error(data: str):
+    buf = Buffer()
+    buf.append_error(data)
+    return buf.output
+
+
+def validate_proto(line: str):
+    """
+    Validate that the line being received is of the expected format.
+
+    Args:
+        line: str containing a JSON document
+
+    Returns: deserialized JSON document
+    """
+    all_types = ["string", "success", "error", "table", "command", "token", "shutdown", "dict"]
+    types_with_data = ["string", "success", "error", "command", "token", "shutdown"]
+    try:
+        json_data = json.loads(line)
+        assert isinstance(json_data, dict)
+        assert "data" in json_data
+        data = json_data["data"]
+        assert isinstance(data, list)
+        for item in data:
+            assert isinstance(item, dict)
+            assert "type" in item
+            it = item["type"]
+            assert it in all_types
+
+            if it in types_with_data:
+                assert "data" in item
+                assert isinstance(item["data"], str)
+            elif it == "table":
+                assert "rows" in item
+                rows = item["rows"]
+                assert isinstance(rows, list)
+                for row in rows:
+                    assert isinstance(row, list)
+
+        return json_data
+    except BaseException:
+        return None
diff --git a/nvflare/fuel/hci/reg.py b/nvflare/fuel/hci/reg.py
new file mode 100644
index 0000000000..53e3dc45dc
--- /dev/null
+++ b/nvflare/fuel/hci/reg.py
@@ -0,0 +1,193 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+
+class CommandSpec(object):
+    """Specification of a command within a CommandModuleSpec to register into CommandRegister as a CommandEntry.
+
+    Args:
+            name: command name
+            description: command description text
+            usage: string to show usage of the command
+            handler_func: function to call for executing the command
+            authz_func: authorization function to run to get a tuple of (valid, authz_ctx) in AuthzFilter
+            visible: whether the command is visible or not
+            confirm: whether the command needs confirmation to execute
+    """
+
+    valid_confirms = ["none", "yesno", "auth"]
+
+    def __init__(
+        self, name: str, description: str, usage: str, handler_func, authz_func=None, visible=True, confirm=None
+    ):
+        self.name = name
+        self.description = description
+        self.usage = usage
+        self.handler_func = handler_func
+        self.authz_func = authz_func
+        self.visible = visible
+        self.confirm = confirm
+        if not confirm:
+            self.confirm = "none"
+        else:
+            assert confirm in CommandSpec.valid_confirms
+
+
+class CommandModuleSpec(object):
+    """
+    Specification for a command module containing a list of commands in the form of CommandSpec.
+
+    Args:
+            name: becomes the scope name of the commands in cmd_specs when registered in CommandRegister
+            cmd_specs: list of CommandSpec objects with
+    """
+
+    def __init__(self, name: str, cmd_specs: List[CommandSpec]):
+        self.name = name
+        self.cmd_specs = cmd_specs
+
+
+class CommandModule(object):
+    """
+    Base class containing CommandModuleSpec.
+    """
+
+    def get_spec(self) -> CommandModuleSpec:
+        """
+
+        Returns: CommandModuleSpec
+
+        """
+        pass
+
+    def close(self):
+        pass
+
+
+class CommandEntry(object):
+    """
+    Contains information about a command. This is registered in Scope within CommandRegister
+
+    Args:
+            scope: scope for this command
+            name: command name
+            desc: command description text
+            usage: string to show usage of the command
+            handler: function to call for executing the command
+            authz_func: authorization function to run to get a tuple of (valid, authz_ctx) in AuthzFilter
+            visible: whether the command is visible or not
+            confirm: whether the command needs confirmation to execute
+    """
+
+    def __init__(self, scope, name, desc, usage, handler, authz_func, visible, confirm):
+        self.scope = scope
+        self.name = name
+        self.desc = desc
+        self.usage = usage
+        self.handler = handler
+        self.authz_func = authz_func
+        self.visible = visible
+        self.confirm = confirm
+
+
+class _Scope(object):
+    """
+    A container grouping CommandEntry objects inside CommandRegister.
+    """
+
+    def __init__(self, name: str):
+        self.name = name
+        self.entries = {}
+
+    def register_command(
+        self, cmd_name: str, cmd_desc: str, cmd_usage: str, handler_func, authz_func, visible, confirm
+    ):
+        self.entries[cmd_name] = CommandEntry(
+            self, cmd_name, cmd_desc, cmd_usage, handler_func, authz_func, visible, confirm
+        )
+
+
+class CommandRegister(object):
+    """
+    Object containing the commands in scopes once they have been registered. ServerCommandRegister is derived from this
+    class and calls the handler of the command through process_command and _do_command. This is also used to register
+    commands for the admin client.
+    """
+
+    def __init__(self, app_ctx):
+        self.app_ctx = app_ctx
+        self.scopes = {}
+        self.cmd_map = {}
+        self.modules = []
+
+    def _get_scope(self, name: str):
+        scope = self.scopes.get(name, None)
+        if scope is None:
+            scope = _Scope(name)
+            self.scopes[name] = scope
+        return scope
+
+    def get_command_entries(self, cmd_name: str):
+        return self.cmd_map.get(cmd_name, [])
+
+    def register_module(self, module: CommandModule, include_invisible=True):
+        self.modules.append(module)
+        module_spec = module.get_spec()
+        for cmd_spec in module_spec.cmd_specs:
+            assert isinstance(cmd_spec, CommandSpec)
+            if cmd_spec.visible or include_invisible:
+                self.add_command(
+                    scope_name=module_spec.name,
+                    cmd_name=cmd_spec.name,
+                    desc=cmd_spec.description,
+                    usage=cmd_spec.usage,
+                    handler=cmd_spec.handler_func,
+                    authz_func=cmd_spec.authz_func,
+                    visible=cmd_spec.visible,
+                    confirm=cmd_spec.confirm,
+                )
+
+    def add_command(self, scope_name, cmd_name, desc, usage, handler, authz_func, visible, confirm):
+        scope = self._get_scope(scope_name)
+        scope.register_command(
+            cmd_name=cmd_name,
+            cmd_desc=desc,
+            cmd_usage=usage,
+            handler_func=handler,
+            authz_func=authz_func,
+            visible=visible,
+            confirm=confirm,
+        )
+
+    def _add_cmd_entry(self, cmd_name, entry):
+        entry_list = self.cmd_map.get(cmd_name, None)
+        if entry_list is None:
+            entry_list = []
+            self.cmd_map[cmd_name] = entry_list
+        entry_list.append(entry)
+
+    def finalize(self, add_cmd_func=None):
+        if len(self.cmd_map) > 0:
+            # already finalized
+            return
+
+        for scope_name, scope in self.scopes.items():
+            for cmd_name, entry in scope.entries.items():
+                self._add_cmd_entry(cmd_name, entry)
+                full_cmd_name = "{}.{}".format(scope_name, cmd_name)
+                self._add_cmd_entry(full_cmd_name, entry)
+                if add_cmd_func:
+                    add_cmd_func(entry)
diff --git a/nvflare/fuel/hci/security.py b/nvflare/fuel/hci/security.py
new file mode 100644
index 0000000000..124f0d2191
--- /dev/null
+++ b/nvflare/fuel/hci/security.py
@@ -0,0 +1,82 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import binascii
+import hashlib
+import os
+import uuid
+
+
+def hash_password(password):
+    """Hash a password for storing.
+
+    Args:
+        password: password to hash
+
+    Returns: hashed password
+
+    """
+    salt = hashlib.sha256(os.urandom(60)).hexdigest().encode("ascii")
+    pwd_hash = hashlib.pbkdf2_hmac(hash_name="sha512", password=password.encode("utf-8"), salt=salt, iterations=100000)
+
+    pwd_hash = binascii.hexlify(pwd_hash)
+    return (salt + pwd_hash).decode("ascii")
+
+
+def verify_password(stored_password, provided_password):
+    """Verify a stored password against one provided by user.
+
+    Args:
+        stored_password: stored password
+        provided_password: password provided by user
+
+    Returns: True if the stored password equals the provided password, otherwise False
+
+    """
+    salt = stored_password[:64]
+    stored_password = stored_password[64:]
+    pwd_hash = hashlib.pbkdf2_hmac(
+        hash_name="sha512", password=provided_password.encode("utf-8"), salt=salt.encode("ascii"), iterations=100000
+    )
+
+    pwd_hash = binascii.hexlify(pwd_hash).decode("ascii")
+    return pwd_hash == stored_password
+
+
+def make_session_token():
+    """Makes a new session token.
+
+    Returns: created session token
+
+    """
+    t = uuid.uuid1()
+    return str(t)
+
+
+def get_certificate_common_name(cert):
+    """ "Gets the common name of the provided certificate.
+
+    Args:
+        cert: certificate
+
+    Returns: common name of provided cert
+
+    """
+    if cert is None:
+        return None
+
+    for sub in cert.get("subject", ()):
+        for key, value in sub:
+            if key == "commonName":
+                return value
diff --git a/nvflare/fuel/hci/server/__init__.py b/nvflare/fuel/hci/server/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/fuel/hci/server/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/fuel/hci/server/audit.py b/nvflare/fuel/hci/server/audit.py
new file mode 100644
index 0000000000..0a23d038ff
--- /dev/null
+++ b/nvflare/fuel/hci/server/audit.py
@@ -0,0 +1,45 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List
+
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.sec.audit import Auditor
+
+from .constants import ConnProps
+from .reg import CommandFilter
+
+
+class CommandAudit(CommandFilter):
+    """
+    Command filter for auditing by adding events.
+
+    This filter needs to be registered after the login filter because it needs the user name established
+    by the login filter.
+    """
+
+    def __init__(self, auditor: Auditor):
+        CommandFilter.__init__(self)
+        assert isinstance(auditor, Auditor), "auditor must be Auditor"
+        self.auditor = auditor
+
+    def pre_command(self, conn: Connection, args: List[str]):
+        user_name = conn.get_prop(ConnProps.USER_NAME, "?")
+
+        event_id = self.auditor.add_event(
+            user=user_name,
+            action=conn.command[:100],  # at most 100 chars
+        )
+
+        conn.set_prop(ConnProps.EVENT_ID, event_id)
+        return True
diff --git a/nvflare/fuel/hci/server/authz.py b/nvflare/fuel/hci/server/authz.py
new file mode 100644
index 0000000000..2ae7ae6e2c
--- /dev/null
+++ b/nvflare/fuel/hci/server/authz.py
@@ -0,0 +1,331 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandEntry, CommandModule, CommandModuleSpec, CommandSpec
+from nvflare.fuel.sec.authz import Authorizer, AuthzContext
+from nvflare.fuel.sec.security_content_service import LoadResult, SecurityContentService
+from nvflare.fuel.utils.time_utils import time_to_string
+
+from .constants import ConnProps
+from .reg import CommandFilter
+
+AUTHORIZATION_POLICY_FILE = "authorization.json"
+
+
+class AuthorizationService(object):
+
+    the_authorizer = None
+
+    @staticmethod
+    def initialize(authorizer: Authorizer, policy_file: str = AUTHORIZATION_POLICY_FILE) -> (Authorizer, str):
+        assert isinstance(authorizer, Authorizer), "authorizer must be Authorizer but got {}".format(type(authorizer))
+
+        if not AuthorizationService.the_authorizer:
+            # get secure content of the policy file
+            policy_conf, result = SecurityContentService.load_json(policy_file)
+            if result == LoadResult.NOT_MANAGED or result == LoadResult.NO_SUCH_CONTENT:
+                # no authorization needed
+                AuthorizationService.the_authorizer = authorizer
+            elif result == LoadResult.OK:
+                err = authorizer.load_policy(policy_conf)
+                if err:
+                    return None, err
+                AuthorizationService.the_authorizer = authorizer
+            else:
+                return None, "invalid policy file {}: {}".format(policy_file, result)
+
+        return AuthorizationService.the_authorizer, ""
+
+    @staticmethod
+    def initialize_with_policy(authorizer: Authorizer, policy_file_path: str) -> (Authorizer, str):
+        assert isinstance(authorizer, Authorizer), "authorizer must be Authorizer but got {}".format(type(authorizer))
+
+        if AuthorizationService.the_authorizer:
+            return AuthorizationService.the_authorizer
+
+        if not os.path.exists(policy_file_path):
+            return None, 'policy file "{}" does not exist'.format(policy_file_path)
+
+        with open(policy_file_path) as file:
+            try:
+                policy_conf = json.load(file)
+            except json.JSONDecodeError:
+                return None, 'policy file "{}" is invalid'.format(policy_file_path)
+
+        err = authorizer.load_policy(policy_conf)
+        if err:
+            return None, err
+
+        AuthorizationService.the_authorizer = authorizer
+        return AuthorizationService.the_authorizer, ""
+
+    @staticmethod
+    def get_authorizer():
+        return AuthorizationService.the_authorizer
+
+    @staticmethod
+    def authorize(ctx: AuthzContext):
+        if not AuthorizationService.the_authorizer:
+            return None, "no authorizer defined"
+        return AuthorizationService.the_authorizer.authorize(ctx)
+
+
+class AuthzFilter(CommandFilter):
+    """
+    Filter for authorization of admin commands.
+    """
+
+    def __init__(self, authorizer: Authorizer):
+        CommandFilter.__init__(self)
+        assert isinstance(authorizer, Authorizer), "authorizer must be Authorizer but got {}".format(type(authorizer))
+        self.authorizer = authorizer
+
+    def pre_command(self, conn: Connection, args: [str]):
+        cmd_entry = conn.get_prop(ConnProps.CMD_ENTRY, None)
+        if not cmd_entry:
+            return True
+
+        assert isinstance(cmd_entry, CommandEntry)
+        authz_func = cmd_entry.authz_func
+        if not authz_func:
+            return True
+
+        valid, authz_ctx = authz_func(conn, args)
+
+        if not valid:
+            return False
+
+        if not authz_ctx or (isinstance(authz_ctx, tuple) and not any(authz_ctx)):
+            # no authz needed
+            return True
+
+        if isinstance(authz_ctx, tuple):
+            for authz in authz_ctx:
+                result = self.check_authz(authz, conn)
+                if not result:
+                    return False
+            return True
+        else:
+            return self.check_authz(authz_ctx, conn)
+
+    def check_authz(self, authz_ctx, conn: Connection):
+        assert isinstance(authz_ctx, AuthzContext), "authz_ctx must be AuthzContext but got {}".format(type(authz_ctx))
+
+        authz_ctx.user_name = conn.get_prop(ConnProps.USER_NAME, "")
+        conn.set_prop(ConnProps.AUTHZ_CTX, authz_ctx)
+        authorized, err = self.authorizer.authorize(ctx=authz_ctx)
+        if err:
+            conn.append_error("Authorization Error: {}".format(err))
+            return False
+
+        if not authorized:
+            conn.append_error("This action is not authorized")
+            return False
+        return True
+
+
+class AuthzCommandModule(CommandModule):
+    def __init__(self, authorizer: Authorizer):
+        assert isinstance(authorizer, Authorizer), "authorizer must be Authorizer but got {}".format(type(authorizer))
+        self.authorizer = authorizer
+
+    def get_spec(self):
+        return CommandModuleSpec(
+            name="authz",
+            cmd_specs=[
+                CommandSpec(
+                    name="show_info",
+                    description="show general info of authorization policy",
+                    usage="show_info",
+                    handler_func=self.show_info,
+                    authz_func=self._authorize_cmd,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="show_users",
+                    description="show users configured for authorization",
+                    usage="show_users",
+                    handler_func=self.show_users,
+                    authz_func=self._authorize_cmd,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="show_sites",
+                    description="show sites configured for authorization",
+                    usage="show_sites",
+                    handler_func=self.show_sites,
+                    authz_func=self._authorize_cmd,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="show_rules",
+                    description="show rules configured for authorization",
+                    usage="show_rules",
+                    handler_func=self.show_rules,
+                    authz_func=self._authorize_cmd,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="show_rights",
+                    description="show rights configured for authorization",
+                    usage="show_rights",
+                    handler_func=self.show_rights,
+                    authz_func=self._authorize_cmd,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="show_config",
+                    description="show authorization config",
+                    usage="show_config",
+                    handler_func=self.show_config,
+                    authz_func=self._authorize_cmd,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="eval_right",
+                    description="check if a user has a right on a site",
+                    usage="eval_right user_name right_name [site_name...]",
+                    handler_func=self.eval_right,
+                    authz_func=self._authorize_cmd,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="eval_rule",
+                    description="evaluate a site against a rule",
+                    usage="eval_rule site_name rule_name",
+                    handler_func=self.eval_rule,
+                    authz_func=self._authorize_cmd,
+                    visible=True,
+                ),
+            ],
+        )
+
+    def _authorize_cmd(self, conn: Connection, args: [str]):
+        # Use this method to pre-process the command,
+        # mainly to get authorizer and policy and store them in conn
+        # so the command handler can get them from the conn.
+        # This way, command handlers don't need to know how to get authorizer or policy.
+        # In case we use different way to get authorizer, this is the only place to change!
+        policy = self.authorizer.get_policy()
+        if not policy:
+            conn.append_error("no authorization policy defined")
+            return False, None
+
+        conn.set_prop("authorizer", self.authorizer)
+        conn.set_prop("policy", policy)
+
+        # return a None authz_ctx because the command does not need to be authorized!
+        return True, None
+
+    def show_info(self, conn: Connection, args: [str]):
+        authorizer = conn.get_prop("authorizer")
+        conn.append_string("Last Loaded: {}".format(time_to_string(authorizer.last_load_time)))
+
+    def show_users(self, conn: Connection, args: [str]):
+        policy = conn.get_prop("policy")
+        users = policy.get_users()
+        table = conn.append_table(["user", "org", "roles"])
+        for user_name, user_def in users.items():
+            table.add_row([user_name, user_def["org"], ",".join(user_def["roles"])])
+
+    def show_sites(self, conn: Connection, args: [str]):
+        policy = conn.get_prop("policy")
+        if not policy:
+            conn.append_error("no authorization policy")
+            return
+
+        sites = policy.get_sites()
+        table = conn.append_table(["site", "org"])
+        for site_name, org in sites.items():
+            table.add_row([site_name, org])
+
+    def show_rights(self, conn: Connection, args: [str]):
+        policy = conn.get_prop("policy")
+        if not policy:
+            conn.append_error("no authorization policy")
+            return
+
+        rights = policy.get_rights()
+        table = conn.append_table(["name", "description", "default"])
+        for name, right_def in rights.items():
+            desc = right_def.get("desc", "")
+            default = right_def.get("default", "")
+            table.add_row([name, desc, "{}".format(default)])
+
+    def show_rules(self, conn: Connection, args: [str]):
+        policy = conn.get_prop("policy")
+        if not policy:
+            conn.append_error("no authorization policy")
+            return
+
+        rules = policy.get_rules()
+        table = conn.append_table(["name", "description", "default"])
+        for name, rule_def in rules.items():
+            desc = rule_def.get("desc", "")
+            default = rule_def.get("default", "")
+            table.add_row([name, desc, "{}".format(default)])
+
+    def show_config(self, conn: Connection, args: [str]):
+        policy = conn.get_prop("policy")
+        config = policy.get_config()
+        conn.append_string(json.dumps(config, indent=1))
+
+    def eval_right(self, conn: Connection, args: [str]):
+        if len(args) < 3:
+            conn.append_error("Usage: {} user_name right_name [site_name...]".format(args[0]))
+            return
+
+        authorizer = conn.get_prop("authorizer")
+        user_name = args[1]
+        right_name = args[2]
+        site_names = args[3:]
+        if not site_names:
+            # all sites
+            policy = conn.get_prop("policy")
+            if not policy:
+                conn.append_error("no authorization policy")
+                return
+
+            sites = policy.get_sites()
+            for s, _ in sites.items():
+                site_names.append(s)
+
+        table = conn.append_table(["Site", "Result"])
+        for s in site_names:
+            result, err = authorizer.evaluate_user_right_on_site(
+                user_name=user_name, site_name=s, right_name=right_name
+            )
+            if err:
+                result = err
+            else:
+                result = str(result)
+            table.add_row([s, result])
+
+    def eval_rule(self, conn: Connection, args: [str]):
+        if len(args) != 3:
+            conn.append_error("Usage: {} site_name rule_name".format(args[0]))
+            return
+
+        authorizer = conn.get_prop("authorizer")
+        site_name = args[1]
+        rule_name = args[2]
+        result, err = authorizer.evaluate_rule_on_site(site_name=site_name, rule_name=rule_name)
+        if err:
+            conn.append_error(err)
+        else:
+            conn.append_string("{}".format(result))
diff --git a/nvflare/fuel/hci/server/builtin.py b/nvflare/fuel/hci/server/builtin.py
new file mode 100644
index 0000000000..a20de9510a
--- /dev/null
+++ b/nvflare/fuel/hci/server/builtin.py
@@ -0,0 +1,98 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List
+
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec
+
+from .reg import ServerCommandRegister
+
+
+class BuiltInCmdModule(CommandModule):
+    """
+    Built in CommandModule with the ability to list commands.
+    """
+
+    def __init__(self, reg: ServerCommandRegister):
+        self.reg = reg
+
+    def get_spec(self):
+        return CommandModuleSpec(
+            name="",
+            cmd_specs=[
+                CommandSpec(
+                    name="_commands",
+                    description="list server commands",
+                    usage="_commands",
+                    handler_func=self.handle_list_commands,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name="echo",
+                    description="echo user input back to client",
+                    usage="echo args ...",
+                    handler_func=self.handle_echo,
+                    visible=False,
+                ),
+            ],
+        )
+
+    def _show_command(self, conn: Connection, cmd_name):
+        entries = self.reg.get_command_entries(cmd_name)
+        if len(entries) <= 0:
+            conn.append_error("undefined command {}\n".format(cmd_name))
+            return
+
+        for e in entries:
+            if not e.visible:
+                continue
+
+            if len(e.scope.name) > 0:
+                conn.append_string("Command: {}.{}".format(e.scope.name, cmd_name))
+            else:
+                conn.append_string("Command: {}".format(cmd_name))
+
+            conn.append_string("Description: {}".format(e.desc))
+            conn.append_string("Usage: {}\n".format(e.usage))
+
+    def handle_list_commands(self, conn: Connection, args: List[str]):
+        if len(args) <= 1:
+            table = conn.append_table(["Scope", "Command", "Description", "Usage", "Confirm"])
+            for scope_name in sorted(self.reg.scopes):
+                scope = self.reg.scopes[scope_name]
+                for cmd_name in sorted(scope.entries):
+                    e = scope.entries[cmd_name]
+                    if e.visible:
+                        table.add_row([scope_name, cmd_name, e.desc, e.usage, e.confirm])
+        else:
+            for cmd_name in args[1:]:
+                self._show_command(conn, cmd_name)
+
+    def handle_echo(self, conn: Connection, args: List[str]):
+        for a in args:
+            conn.append_string(a)
+
+
+def new_command_register_with_builtin_module(app_ctx):
+    """Creates ServerCommandRegister and registers builtin command module.
+
+    Args:
+        app_ctx: engine
+
+    Returns: ServerCommandRegister
+
+    """
+    reg = ServerCommandRegister(app_ctx=app_ctx)
+    reg.register_module(BuiltInCmdModule(reg))
+    return reg
diff --git a/nvflare/fuel/hci/server/constants.py b/nvflare/fuel/hci/server/constants.py
new file mode 100644
index 0000000000..578e8079f6
--- /dev/null
+++ b/nvflare/fuel/hci/server/constants.py
@@ -0,0 +1,26 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class ConnProps(object):
+    """
+    Constants for connection properties.
+    """
+
+    EVENT_ID = "_eventId"
+    USER_NAME = "_userName"
+    TOKEN = "_sessionToken"
+    SESSION = "_session"
+    CMD_ENTRY = "_cmdEntry"
+    AUTHZ_CTX = "_authztx"
diff --git a/nvflare/fuel/hci/server/file_transfer.py b/nvflare/fuel/hci/server/file_transfer.py
new file mode 100644
index 0000000000..d60f803239
--- /dev/null
+++ b/nvflare/fuel/hci/server/file_transfer.py
@@ -0,0 +1,239 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tempfile
+import traceback
+
+import nvflare.fuel.hci.file_transfer_defs as ftd
+from nvflare.fuel.hci.base64_utils import (
+    b64str_to_binary_file,
+    b64str_to_bytes,
+    b64str_to_text_file,
+    binary_file_to_b64str,
+    bytes_to_b64str,
+    text_file_to_b64str,
+)
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec
+from nvflare.fuel.hci.server.authz import AuthzContext
+from nvflare.fuel.hci.zip_utils import unzip_all_from_bytes, zip_directory_to_bytes
+
+
+def upload_folder_authz_func_signature(folder_path: str) -> (str, AuthzContext):
+    pass
+
+
+class FileTransferModule(CommandModule):
+    """
+    Command module for file transfers.
+    """
+
+    def __init__(self, upload_dir: str, download_dir: str, upload_folder_authz_func=None):
+        assert os.path.isdir(upload_dir), "upload_dir {} is not a valid dir".format(upload_dir)
+
+        assert os.path.isdir(download_dir), "download_dir {} is not a valid dir".format(download_dir)
+
+        self.upload_dir = upload_dir
+        self.download_dir = download_dir
+        self.upload_folder_authz_func = upload_folder_authz_func
+
+    def get_spec(self):
+        return CommandModuleSpec(
+            name=ftd.SERVER_MODULE_NAME,
+            cmd_specs=[
+                CommandSpec(
+                    name=ftd.SERVER_CMD_UPLOAD_TEXT,
+                    description="upload one or more text files",
+                    usage="_upload name1 data1 name2 data2 ...",
+                    handler_func=self.upload_text_file,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name=ftd.SERVER_CMD_DOWNLOAD_TEXT,
+                    description="download one or more text files",
+                    usage="download file_name ...",
+                    handler_func=self.download_text_file,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name=ftd.SERVER_CMD_UPLOAD_BINARY,
+                    description="upload one or more binary files",
+                    usage="upload name1 data1 name2 data2 ...",
+                    handler_func=self.upload_binary_file,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name=ftd.SERVER_CMD_DOWNLOAD_BINARY,
+                    description="download one or more binary files",
+                    usage="download file_name ...",
+                    handler_func=self.download_binary_file,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name=ftd.SERVER_CMD_UPLOAD_FOLDER,
+                    description="upload a folder from client",
+                    usage="upload_folder folder_name",
+                    handler_func=self.upload_folder,
+                    authz_func=self._authorize_upload_folder,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name=ftd.SERVER_CMD_DOWNLOAD_FOLDER,
+                    description="download a folder to client",
+                    usage="download folder_name",
+                    handler_func=self.download_folder,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name=ftd.SERVER_CMD_INFO,
+                    description="show info",
+                    usage="info",
+                    handler_func=self.info,
+                    visible=False,
+                ),
+            ],
+        )
+
+    def upload_file(self, conn: Connection, args: [str], str_to_file_func):
+        if len(args) < 3:
+            conn.append_error("syntax error: missing files")
+            return
+
+        if len(args) % 2 != 1:
+            conn.append_error("syntax error: file name/data not paired")
+            return
+
+        table = conn.append_table(["file", "size"])
+        i = 1
+        while i < len(args):
+            name = args[i]
+            data = args[i + 1]
+            i += 2
+
+            full_path = os.path.join(self.upload_dir, name)
+            num_bytes = str_to_file_func(b64str=data, file_name=full_path)
+            table.add_row([name, str(num_bytes)])
+
+    def upload_text_file(self, conn: Connection, args: [str]):
+        self.upload_file(conn, args, b64str_to_text_file)
+
+    def upload_binary_file(self, conn: Connection, args: [str]):
+        self.upload_file(conn, args, b64str_to_binary_file)
+
+    def download_file(self, conn: Connection, args: [str], file_to_str_func):
+        if len(args) < 2:
+            conn.append_error("syntax error: missing file names")
+            return
+
+        table = conn.append_table(["name", "data"])
+        for i in range(1, len(args)):
+            file_name = args[i]
+            full_path = os.path.join(self.download_dir, file_name)
+            if not os.path.exists(full_path):
+                conn.append_error("no such file: {}".format(file_name))
+                continue
+
+            if not os.path.isfile(full_path):
+                conn.append_error("not a file: {}".format(file_name))
+                continue
+
+            encoded_str = file_to_str_func(full_path)
+            table.add_row([file_name, encoded_str])
+
+    def download_text_file(self, conn: Connection, args: [str]):
+        self.download_file(conn, args, text_file_to_b64str)
+
+    def download_binary_file(self, conn: Connection, args: [str]):
+        self.download_file(conn, args, binary_file_to_b64str)
+
+    def _authorize_upload_folder(self, conn: Connection, args: [str]):
+        if len(args) != 3:
+            conn.append_error("syntax error: require data")
+            return False, None
+
+        folder_name = args[1]
+        zip_b64str = args[2]
+        tmp_dir = tempfile.mkdtemp()
+
+        try:
+            data_bytes = b64str_to_bytes(zip_b64str)
+            unzip_all_from_bytes(data_bytes, tmp_dir)
+            tmp_folder_path = os.path.join(tmp_dir, folder_name)
+
+            if not os.path.isdir(tmp_folder_path):
+                conn.append_error("logic error: unzip failed to create folder {}".format(tmp_folder_path))
+                return False, None
+
+            if self.upload_folder_authz_func:
+                err, authz_ctx = self.upload_folder_authz_func(tmp_folder_path)
+                if err is None:
+                    err = ""
+                elif not isinstance(err, str):
+                    # the validator failed to follow signature
+                    # assuming things are bad
+                    err = "folder validation failed"
+
+                if len(err) > 0:
+                    conn.append_error(err)
+                    return False, None
+                else:
+                    return True, authz_ctx
+            else:
+                return True, None
+        except BaseException:
+            traceback.print_exc()
+            conn.append_error("exception occurred")
+            return False, None
+        finally:
+            shutil.rmtree(tmp_dir)
+
+    def upload_folder(self, conn: Connection, args: [str]):
+        folder_name = args[1]
+        zip_b64str = args[2]
+        folder_path = os.path.join(self.upload_dir, folder_name)
+        if os.path.exists(folder_path):
+            shutil.rmtree(folder_path)
+        data_bytes = b64str_to_bytes(zip_b64str)
+        unzip_all_from_bytes(data_bytes, self.upload_dir)
+        conn.set_prop("upload_folder_path", folder_path)
+        conn.append_string("Created folder {}".format(folder_path))
+
+    def download_folder(self, conn: Connection, args: [str]):
+        if len(args) != 2:
+            conn.append_error("syntax error: require folder name")
+            return
+
+        folder_name = args[1]
+        full_path = os.path.join(self.download_dir, folder_name)
+        if not os.path.exists(full_path):
+            conn.append_error("no such folder: {}".format(full_path))
+            return
+
+        if not os.path.isdir(full_path):
+            conn.append_error("'{}' is not a valid folder".format(full_path))
+            return
+
+        try:
+            data = zip_directory_to_bytes(self.download_dir, folder_name)
+            b64str = bytes_to_b64str(data)
+            conn.append_string(b64str)
+        except BaseException:
+            traceback.print_exc()
+            conn.append_error("exception occurred")
+
+    def info(self, conn: Connection, args: [str]):
+        conn.append_string("Server Upload Destination: {}".format(self.upload_dir))
+        conn.append_string("Server Download Source: {}".format(self.download_dir))
diff --git a/nvflare/fuel/hci/server/hci.py b/nvflare/fuel/hci/server/hci.py
new file mode 100644
index 0000000000..40b63179a8
--- /dev/null
+++ b/nvflare/fuel/hci/server/hci.py
@@ -0,0 +1,164 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import socketserver
+import ssl
+import threading
+import traceback
+
+from nvflare.fuel.hci.conn import Connection, receive_til_end
+from nvflare.fuel.hci.proto import validate_proto
+from nvflare.fuel.hci.security import get_certificate_common_name
+
+from .reg import ServerCommandRegister
+
+
+class _MsgHandler(socketserver.BaseRequestHandler):
+    """
+    Message handler used by the AdminServer to receive admin commands, validate, then process and do command through the
+    ServerCommandRegister.
+    """
+
+    def handle(self):
+        try:
+            conn = Connection(self.request, self.server)
+
+            if self.server.use_ssl:
+                cn = get_certificate_common_name(self.request.getpeercert())
+                conn.set_prop("_client_cn", cn)
+                valid = self.server.validate_client_cn(cn)
+            else:
+                valid = True
+
+            if not valid:
+                conn.append_error("authentication error")
+            else:
+                req = receive_til_end(self.request).strip()
+                command = None
+                req_json = validate_proto(req)
+                conn.request = req_json
+                if req_json is not None:
+                    data = req_json["data"]
+                    for item in data:
+                        it = item["type"]
+                        if it == "command":
+                            command = item["data"]
+                            break
+
+                    if command is None:
+                        conn.append_error("protocol violation")
+                    else:
+                        self.server.cmd_reg.process_command(conn, command)
+                else:
+                    # not json encoded
+                    conn.append_error("protocol violation")
+
+            if not conn.ended:
+                conn.close()
+        except BaseException:
+            traceback.print_exc()
+
+
+def initialize_hci():
+    socketserver.TCPServer.allow_reuse_address = True
+
+
+class AdminServer(socketserver.ThreadingTCPServer):
+    """
+    Base class of FedAdminServer to create a server that can receive commands.
+    """
+
+    # faster re-binding
+    allow_reuse_address = True
+
+    # make this bigger than five
+    request_queue_size = 10
+
+    # kick connections when we exit
+    daemon_threads = True
+
+    def __init__(
+        self,
+        cmd_reg: ServerCommandRegister,
+        host,
+        port,
+        ca_cert=None,
+        server_cert=None,
+        server_key=None,
+        accepted_client_cns=None,
+    ):
+
+        socketserver.TCPServer.__init__(self, (host, port), _MsgHandler, False)
+
+        self.use_ssl = False
+        if ca_cert and server_cert:
+            if accepted_client_cns:
+                assert isinstance(accepted_client_cns, list), "accepted_client_cns must be list"
+
+            ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+            ctx.verify_mode = ssl.CERT_REQUIRED
+            ctx.load_verify_locations(ca_cert)
+            ctx.load_cert_chain(certfile=server_cert, keyfile=server_key)
+
+            # replace the socket with an ssl version of itself
+            self.socket = ctx.wrap_socket(self.socket, server_side=True)
+            self.use_ssl = True
+
+        # bind the socket and start the server
+        self.server_bind()
+        self.server_activate()
+
+        self._thread = None
+        self.host = host
+        self.port = port
+        self.accepted_client_cns = accepted_client_cns
+        self.cmd_reg = cmd_reg
+        cmd_reg.finalize()
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def validate_client_cn(self, cn):
+        if self.accepted_client_cns:
+            return cn in self.accepted_client_cns
+        else:
+            return True
+
+    def stop(self):
+        self.shutdown()
+        self.cmd_reg.close()
+
+        if self._thread.is_alive():
+            self._thread.join()
+
+        self.logger.info(f"Admin Server {self.host} on Port {self.port} shutdown!")
+
+    def set_command_registry(self, cmd_reg: ServerCommandRegister):
+        if cmd_reg:
+            cmd_reg.finalize()
+
+            if self.cmd_reg:
+                self.cmd_reg.close()
+
+            self.cmd_reg = cmd_reg
+
+    def start(self):
+        if self._thread is None:
+            self._thread = threading.Thread(target=self._run, args=())
+
+        if not self._thread.is_alive():
+            self._thread.start()
+
+    def _run(self):
+        self.logger.info(f"Starting Admin Server {self.host} on Port {self.port}")
+        self.serve_forever()
diff --git a/nvflare/fuel/hci/server/login.py b/nvflare/fuel/hci/server/login.py
new file mode 100644
index 0000000000..cabf5b4f3e
--- /dev/null
+++ b/nvflare/fuel/hci/server/login.py
@@ -0,0 +1,200 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec
+from nvflare.fuel.hci.security import verify_password
+from nvflare.fuel.hci.server.constants import ConnProps
+
+from .reg import CommandFilter
+from .sess import SessionManager
+
+LOGIN_CMD_NAME = "_login"
+CERT_LOGIN_CMD_NAME = "_cert_login"
+
+
+class Authenticator(object):
+    """
+    Base class for authenticating credentials.
+    """
+
+    def authenticate(self, user_name: str, credential: str, credential_type: str) -> bool:
+        """
+        Authenticate a specified user with the provided credential.
+
+        Args:
+            user_name: user login name
+            credential: provided credential
+            credential_type: type of credential
+
+        Returns: True if successful, False otherwise
+
+        """
+        pass
+
+
+class SimpleAuthenticator(Authenticator):
+    """
+    Authenticator to use in the LoginModule for authenticating admin clients for login.
+    """
+
+    def __init__(self, users):
+        self.users = users
+
+    def authenticate_password(self, user_name: str, pwd: str):
+        pwd_hash = self.users.get(user_name)
+        if pwd_hash is None:
+            return False
+
+        return verify_password(pwd_hash, pwd)
+
+    def authenticate_cn(self, user_name: str, cn):
+        return user_name == cn
+
+    def authenticate(self, user_name: str, credential, credential_type):
+        if credential_type == "password":
+            return self.authenticate_password(user_name, credential)
+        elif credential_type == "cn":
+            return self.authenticate_cn(user_name, credential)
+        else:
+            return False
+
+
+class LoginModule(CommandModule, CommandFilter):
+    """
+    CommandModule containing the login commands to handle login and logout of admin clients, as well as the
+    CommandFilter pre_command to check that a client is logged in with a valid session.
+    """
+
+    def __init__(self, authenticator: Authenticator, sess_mgr: SessionManager):
+        if authenticator:
+            assert isinstance(authenticator, Authenticator), "authenticator must be Authenticator"
+
+        assert isinstance(sess_mgr, SessionManager), "sess_mgr must be SessionManager"
+
+        self.authenticator = authenticator
+        self.session_mgr = sess_mgr
+
+    def get_spec(self):
+        return CommandModuleSpec(
+            name="login",
+            cmd_specs=[
+                CommandSpec(
+                    name=LOGIN_CMD_NAME,
+                    description="login to server",
+                    usage="login userName password",
+                    handler_func=self.handle_login,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name=CERT_LOGIN_CMD_NAME,
+                    description="login to server with SSL cert",
+                    usage="login userName",
+                    handler_func=self.handle_cert_login,
+                    visible=False,
+                ),
+                CommandSpec(
+                    name="_logout",
+                    description="logout from server",
+                    usage="logout",
+                    handler_func=self.handle_logout,
+                    visible=False,
+                ),
+            ],
+        )
+
+    def handle_login(self, conn: Connection, args: List[str]):
+        if not self.authenticator:
+            conn.append_string("OK")
+            return
+
+        if len(args) != 3:
+            conn.append_string("REJECT")
+            return
+
+        user_name = args[1]
+        pwd = args[2]
+
+        ok = self.authenticator.authenticate(user_name, pwd, "password")
+        if not ok:
+            conn.append_string("REJECT")
+            return
+
+        session = self.session_mgr.create_session(user_name)
+        conn.append_string("OK")
+        conn.append_token(session.token)
+
+    def handle_cert_login(self, conn: Connection, args: List[str]):
+        if not self.authenticator:
+            conn.append_string("OK")
+            return
+
+        if len(args) != 2:
+            conn.append_string("REJECT")
+            return
+
+        cn = conn.get_prop("_client_cn", None)
+        if cn is None:
+            conn.append_string("REJECT")
+            return
+
+        user_name = args[1]
+
+        ok = self.authenticator.authenticate(user_name, cn, "cn")
+        if not ok:
+            conn.append_string("REJECT")
+            return
+
+        session = self.session_mgr.create_session(user_name)
+        conn.append_string("OK")
+        conn.append_token(session.token)
+
+    def handle_logout(self, conn: Connection, args: List[str]):
+        if self.authenticator and self.session_mgr:
+            token = conn.get_prop(ConnProps.TOKEN)
+            if token:
+                self.session_mgr.end_session(token)
+        conn.append_string("OK")
+
+    def pre_command(self, conn: Connection, args: List[str]):
+        if args[0] in [LOGIN_CMD_NAME, CERT_LOGIN_CMD_NAME]:
+            # skip login command
+            return True
+
+        # validate token
+        req_json = conn.request
+        token = None
+        data = req_json["data"]
+        for item in data:
+            it = item["type"]
+            if it == "token":
+                token = item["data"]
+                break
+
+        if token is None:
+            conn.append_error("not authenticated - no token")
+            return False
+
+        sess = self.session_mgr.get_session(token)
+        if sess:
+            sess.mark_active()
+            conn.set_prop(ConnProps.SESSION, sess)
+            conn.set_prop(ConnProps.USER_NAME, sess.user_name)
+            conn.set_prop(ConnProps.TOKEN, token)
+            return True
+        else:
+            conn.append_error("not authenticated - no user")
+            return False
diff --git a/nvflare/fuel/hci/server/reg.py b/nvflare/fuel/hci/server/reg.py
new file mode 100644
index 0000000000..7eee01dc26
--- /dev/null
+++ b/nvflare/fuel/hci/server/reg.py
@@ -0,0 +1,119 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import traceback
+from typing import List
+
+from nvflare.fuel.hci.cmd_arg_utils import split_to_args
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandRegister
+
+from .constants import ConnProps
+
+
+class CommandFilter(object):
+    """
+    Base class for filters to run before or after commands.
+    """
+
+    def pre_command(self, conn: Connection, args: List[str]) -> bool:
+        """Code to execute before executing a command.
+
+        Returns: True to continue filter chain or False to not
+        """
+        return True
+
+    def post_command(self, conn: Connection, args: List[str]) -> bool:
+        """Code to execute after executing a command."""
+        pass
+
+    def close(self):
+        pass
+
+
+class ServerCommandRegister(CommandRegister):
+    """Runs filters and executes commands by calling their handler function. This is the main command register used by
+    AdminServer."""
+
+    def __init__(self, app_ctx):
+        CommandRegister.__init__(self, app_ctx)
+        self.filters = []
+        self.closed = False
+
+    def add_filter(self, cmd_filter: CommandFilter):
+        assert isinstance(cmd_filter, CommandFilter), "cmd_filter must be CommandFilter"
+        self.filters.append(cmd_filter)
+
+    def _do_command(self, conn: Connection, command: str):
+        """
+        Getting the command from the command registry, invoke filters and call the handler function, passing along conn
+        and the args split from the command.
+        """
+        conn.app_ctx = self.app_ctx
+        args = split_to_args(command)
+        conn.args = args
+        conn.command = command
+
+        cmd_name = args[0]
+        entries = self.get_command_entries(cmd_name)
+        if len(entries) <= 0:
+            conn.append_error('Unknown command "{}"'.format(cmd_name))
+            return
+        elif len(entries) == 1:
+            conn.set_prop(ConnProps.CMD_ENTRY, entries[0])
+            handler = entries[0].handler
+        else:
+            conn.append_error('Command "{}" exists in multiple scopes. Please use full command name'.format(cmd_name))
+            return
+
+        if handler is None:
+            conn.append_error('Unknown command "{}"'.format(cmd_name))
+            return
+
+        # invoke pre filters
+        if len(self.filters) > 0:
+            for f in self.filters:
+                ok = f.pre_command(conn, args)
+                if not ok:
+                    return
+
+        if handler is not None:
+            handler(conn, args)
+        else:
+            conn.append_error('Unknown command "{}"'.format(command))
+            return
+
+        # invoke post filters
+        if len(self.filters) > 0:
+            for f in self.filters:
+                f.post_command(conn, args)
+
+    def process_command(self, conn: Connection, command: str):
+        try:
+            self._do_command(conn, command)
+        except BaseException:
+            traceback.print_exc()
+            conn.append_error("Exception Occurred")
+
+    def close(self):
+        if self.closed:
+            return
+
+        for f in self.filters:
+            f.close()
+
+        for m in self.modules:
+            m.close()
+
+        self.closed = True
diff --git a/nvflare/fuel/hci/server/sess.py b/nvflare/fuel/hci/server/sess.py
new file mode 100644
index 0000000000..9500c15a2f
--- /dev/null
+++ b/nvflare/fuel/hci/server/sess.py
@@ -0,0 +1,151 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+from typing import List
+
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec
+from nvflare.fuel.hci.security import make_session_token
+from nvflare.fuel.utils.time_utils import time_to_string
+
+
+class Session(object):
+    """
+    Object keeping track of an admin client session with token and time data.
+    """
+
+    def __init__(self):
+        self.user_name = None
+        self.start_time = None
+        self.last_active_time = None
+        self.token = None
+
+    def mark_active(self):
+        self.last_active_time = time.time()
+
+
+class SessionManager(CommandModule):
+    """
+    CommandModule handling session management.
+    """
+
+    def __init__(self, idle_timeout=3600, monitor_interval=5):
+        if monitor_interval <= 0:
+            monitor_interval = 5
+
+        self.sess_update_lock = threading.Lock()
+        self.sessions = {}  # token => Session
+        self.idle_timeout = idle_timeout
+        self.monitor_interval = monitor_interval
+        self.asked_to_stop = False
+        self.monitor = threading.Thread(target=self.monitor_sessions)
+        self.monitor.start()
+
+    def monitor_sessions(self):
+        """Runs loop in a thread to end sessions that time out."""
+        while True:
+            # print('checking for dead sessions ...')
+            if self.asked_to_stop:
+                break
+
+            dead_sess = None
+            for _, sess in self.sessions.items():
+                time_passed = time.time() - sess.last_active_time
+                # print('time passed: {} secs'.format(time_passed))
+                if time_passed > self.idle_timeout:
+                    dead_sess = sess
+                    break
+
+            if dead_sess:
+                # print('ending dead session {}'.format(dead_sess.token))
+                self.end_session(dead_sess.token)
+            else:
+                # print('no dead sessions found')
+                pass
+
+            time.sleep(self.monitor_interval)
+
+    def shutdown(self):
+        self.asked_to_stop = True
+        self.monitor.join(timeout=10)
+
+    def create_session(self, user_name):
+        """
+        Creates new session with a new session token.
+
+        Args:
+            user_name: user name for session
+
+        Returns: Session
+
+        """
+        token = make_session_token()
+        sess = Session()
+        sess.user_name = user_name
+        sess.start_time = time.time()
+        sess.last_active_time = sess.start_time
+        sess.token = token
+        with self.sess_update_lock:
+            self.sessions[token] = sess
+        return sess
+
+    def get_session(self, token: str):
+        with self.sess_update_lock:
+            return self.sessions.get(token)
+
+    def get_sessions(self):
+        result = []
+        with self.sess_update_lock:
+            for _, s in self.sessions.items():
+                result.append(s)
+        return result
+
+    def end_session(self, token):
+        with self.sess_update_lock:
+            self.sessions.pop(token, None)
+
+    def get_spec(self):
+        return CommandModuleSpec(
+            name="sess",
+            cmd_specs=[
+                CommandSpec(
+                    name="list_sessions",
+                    description="list user sessions",
+                    usage="list_sessions",
+                    handler_func=self.handle_list_sessions,
+                    visible=True,
+                )
+            ],
+        )
+
+    def handle_list_sessions(self, conn: Connection, args: List[str]):
+        """
+        Lists sessions and the details in a table. Not registered by default but can be registered in FedAdminServer
+        with cmd_reg.register_module(sess_mgr).
+        """
+        sess_list = list(self.sessions.values())
+        sess_list.sort(key=lambda x: x.user_name, reverse=False)
+        table = conn.append_table(["User", "Session ID", "Start", "Last Active", "Idle"])
+        for s in sess_list:
+            table.add_row(
+                [
+                    s.user_name,
+                    "{}".format(s.token),
+                    time_to_string(s.start_time),
+                    time_to_string(s.last_active_time),
+                    "{}".format(time.time() - s.last_active_time),
+                ]
+            )
diff --git a/nvflare/fuel/hci/shell_cmd_val.py b/nvflare/fuel/hci/shell_cmd_val.py
new file mode 100644
index 0000000000..1cdc0d52fc
--- /dev/null
+++ b/nvflare/fuel/hci/shell_cmd_val.py
@@ -0,0 +1,106 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from nvflare.fuel.hci.cmd_arg_utils import ArgValidator
+
+
+class ShellCommandValidator(object):
+    """
+    Base class for validators to be called by command executors for shell commands.
+    """
+
+    def __init__(self, arg_validator: ArgValidator):
+        self.arg_validator = arg_validator
+
+    def validate(self, args: List[str]):
+        self.arg_validator.err = ""
+        return self.arg_validator.validate(args)
+
+    def get_usage(self):
+        return self.arg_validator.get_usage()
+
+
+class TailValidator(ShellCommandValidator):
+    """
+    Validator for the tail command.
+    """
+
+    def __init__(self):
+        val = ArgValidator("tail")
+        val.add_argument("-c", type=int, help="output the last C bytes")
+        val.add_argument("-n", type=int, help="output the last N lines")
+        val.add_argument("files", metavar="file", type=str, nargs="+")
+        ShellCommandValidator.__init__(self, val)
+
+
+class HeadValidator(ShellCommandValidator):
+    """
+    Validator for the head command.
+    """
+
+    def __init__(self):
+        val = ArgValidator("head")
+        val.add_argument("-c", type=int, help="print the first C bytes of each file")
+        val.add_argument("-n", type=int, help="print the first N lines instead of the first 10")
+        val.add_argument("files", metavar="file", type=str, nargs="+")
+        ShellCommandValidator.__init__(self, val)
+
+
+class GrepValidator(ShellCommandValidator):
+    """
+    Validator for the grep command.
+    """
+
+    def __init__(self):
+        val = ArgValidator("grep")
+        val.add_argument("-n", action="store_true", help="print line number with output lines")
+        val.add_argument("-i", action="store_true", help="ignore case distinctions")
+        val.add_argument("-b", action="store_true", help="print the byte offset with output lines")
+        val.add_argument("pattern", metavar="pattern", type=str)
+        val.add_argument("files", metavar="file", type=str, nargs="+")
+        ShellCommandValidator.__init__(self, val)
+
+
+class CatValidator(ShellCommandValidator):
+    """
+    Validator for the cat command.
+    """
+
+    def __init__(self):
+        val = ArgValidator("cat")
+        val.add_argument("-n", action="store_true", help="number all output lines")
+        val.add_argument("-b", action="store_true", help="number nonempty output lines, overrides -n")
+        val.add_argument("-s", action="store_true", help="suppress repeated empty output lines")
+        val.add_argument("-T", action="store_true", help="display TAB characters as ^I")
+        val.add_argument("files", metavar="file", type=str, nargs="+")
+        ShellCommandValidator.__init__(self, val)
+
+
+class LsValidator(ShellCommandValidator):
+    """
+    Validator for the ls command.
+    """
+
+    def __init__(self):
+        val = ArgValidator("ls")
+        val.add_argument("-a", action="store_true")
+        val.add_argument("-l", action="store_true", help="use a long listing format")
+        val.add_argument("-t", action="store_true", help="sort by modification time, newest first")
+        val.add_argument("-S", action="store_true", help="sort by file size, largest first")
+        val.add_argument("-R", action="store_true", help="list subdirectories recursively")
+        val.add_argument("-u", action="store_true", help="with -l: show access time, otherwise: sort by access time")
+        val.add_argument("files", metavar="file", type=str, nargs="?")
+        ShellCommandValidator.__init__(self, val)
diff --git a/nvflare/fuel/hci/table.py b/nvflare/fuel/hci/table.py
new file mode 100644
index 0000000000..1eff39276a
--- /dev/null
+++ b/nvflare/fuel/hci/table.py
@@ -0,0 +1,102 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List
+
+
+def repeat_to_length(string_to_expand, length):
+    """Repeats string_to_expand to fill up a string of the provided length.
+
+    Args:
+        string_to_expand: string to repeat
+        length: length of string to return
+
+    Returns: generated string of provided length
+
+    """
+    return (string_to_expand * (int(length / len(string_to_expand)) + 1))[:length]
+
+
+class Table(object):
+    """
+    Table structure to write to. The headers are set in the init and the data is input to rows before write
+    writes the table.
+
+    Args:
+            headers: headers of the table
+    """
+
+    def __init__(self, headers: List[str]):
+        self.rows = []
+        if headers and len(headers) > 0:
+            new_headers = []
+            for h in headers:
+                new_headers.append(h.upper())
+            self.rows.append(new_headers)
+
+    def set_rows(self, rows):
+        self.rows = rows
+
+    def add_row(self, row: [str]):
+        self.rows.append(row)
+
+    def write(self, writer):
+        # compute the number of cols
+        num_cols = 0
+        for row in self.rows:
+            if num_cols < len(row):
+                num_cols = len(row)
+
+        # compute max col size
+        col_len = [0 for _ in range(num_cols)]
+        for row in self.rows:
+            for i in range(len(row)):
+                if col_len[i] < len(row[i]):
+                    col_len[i] = len(row[i])
+
+        col_fmt = ["" for _ in range(num_cols)]
+        for i in range(num_cols):
+            if i == 0:
+                extra = ""
+            else:
+                extra = " "
+
+            col_fmt[i] = extra + "| {:" + "{}".format(col_len[i]) + "}"
+            if i == num_cols - 1:
+                col_fmt[i] = col_fmt[i] + " |"
+
+        total_col_size = 0
+        for i in range(num_cols):
+            total_col_size += col_len[i] + 2
+
+        table_width = total_col_size + num_cols + 1
+        border_line = repeat_to_length("-", table_width)
+        writer.write(border_line + "\n")
+
+        for r in range(len(self.rows)):
+            row = self.rows[r]
+            line = ""
+            for i in range(num_cols):
+                if i < len(row):
+                    data = row[i]
+                else:
+                    data = " "
+
+                line += col_fmt[i].format(data)
+
+            writer.write(line + "\n")
+
+            if r == 0:
+                writer.write(border_line + "\n")
+
+        writer.write(border_line + "\n")
diff --git a/nvflare/fuel/hci/tools/__init__.py b/nvflare/fuel/hci/tools/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/fuel/hci/tools/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/fuel/hci/tools/admin.py b/nvflare/fuel/hci/tools/admin.py
new file mode 100644
index 0000000000..d24ba55f52
--- /dev/null
+++ b/nvflare/fuel/hci/tools/admin.py
@@ -0,0 +1,104 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+
+from nvflare.fuel.hci.client.file_transfer import FileTransferModule
+from nvflare.fuel.hci.client.lib import AdminClient
+
+
+def main():
+    """
+    Script to launch the admin client to issue admin commands to the server.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", default="localhost")
+    parser.add_argument("--port", type=int, default=55550)
+    parser.add_argument("--prompt", type=str, default="> ")
+    parser.add_argument("--with_file_transfer", action="store_true")
+    parser.add_argument("--upload_folder_cmd_name", type=str, default="upload_app")
+    parser.add_argument("--upload_dir", type=str, default="")
+    parser.add_argument("--download_dir", type=str, default="")
+    parser.add_argument("--with_shell", action="store_true")
+    parser.add_argument("--with_login", action="store_true")
+    parser.add_argument("--cred_type", default="password")
+    parser.add_argument("--with_ssl", action="store_true")
+    parser.add_argument("--ca_cert", type=str, default="")
+    parser.add_argument("--client_cert", type=str, default="")
+    parser.add_argument("--client_key", type=str, default="")
+    parser.add_argument("--with_debug", action="store_true")
+
+    args = parser.parse_args()
+
+    modules = []
+
+    if args.with_file_transfer:
+        modules.append(
+            FileTransferModule(
+                upload_dir=args.upload_dir,
+                download_dir=args.download_dir,
+                upload_folder_cmd_name=args.upload_folder_cmd_name,
+            )
+        )
+
+    ca_cert = args.ca_cert
+    client_cert = args.client_cert
+    client_key = args.client_key
+
+    if args.with_ssl:
+        if len(ca_cert) <= 0:
+            print("missing CA Cert file name")
+            return
+
+        if len(client_cert) <= 0:
+            print("missing Client Cert file name")
+            return
+
+        if len(client_key) <= 0:
+            print("missing Client Key file name")
+            return
+    else:
+        ca_cert = None
+        client_key = None
+        client_cert = None
+
+    if args.with_debug:
+        print("SSL: {}".format(args.with_ssl))
+        print("User Login: {}".format(args.with_login))
+        print("File Transfer: {}".format(args.with_file_transfer))
+
+        if args.with_file_transfer:
+            print("  Upload Dir: {}".format(args.upload_dir))
+            print("  Download Dir: {}".format(args.download_dir))
+
+    print("Admin Server: {} on port {}".format(args.host, args.port))
+
+    client = AdminClient(
+        host=args.host,
+        port=args.port,
+        prompt=args.prompt,
+        cmd_modules=modules,
+        ca_cert=ca_cert,
+        client_cert=client_cert,
+        client_key=client_key,
+        require_login=args.with_login,
+        credential_type=args.cred_type,
+        debug=args.with_debug,
+    )
+
+    client.run()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/nvflare/fuel/hci/tools/authz_preview.py b/nvflare/fuel/hci/tools/authz_preview.py
new file mode 100644
index 0000000000..bdd06d2964
--- /dev/null
+++ b/nvflare/fuel/hci/tools/authz_preview.py
@@ -0,0 +1,223 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import cmd
+import json
+
+from nvflare.fuel.hci.cmd_arg_utils import split_to_args
+from nvflare.fuel.hci.table import Table
+from nvflare.fuel.sec.authz import Policy, validate_policy_config
+
+
+class Commander(cmd.Cmd):
+    """
+    Command line prompt helper tool for getting information for authorization configurations.
+    Args:
+            policy: authorization policy object
+    """
+
+    def __init__(self, policy: Policy):
+        cmd.Cmd.__init__(self)
+        self.policy = policy
+        self.intro = "Type help or ? to list commands.\n"
+        self.prompt = ">"
+
+    def do_bye(self, arg):
+        """exit from the client"""
+        return True
+
+    def emptyline(self):
+        return
+
+    def do_show_users(self, arg):
+        users = self.policy.get_users()
+        table = Table(["user", "org", "roles"])
+        for user_name, user_def in users.items():
+            table.add_row([user_name, user_def["org"], ",".join(user_def["roles"])])
+        self.write_table(table)
+
+    def _split_to_args(self, arg):
+        if len(arg) <= 0:
+            return []
+        else:
+            return split_to_args(arg)
+
+    def do_show_sites(self, arg):
+        sites = self.policy.get_sites()
+        table = Table(["site", "org"])
+        for site_name, org in sites.items():
+            table.add_row([site_name, org])
+        self.write_table(table)
+
+    def do_show_rights(self, arg):
+        rights = self.policy.get_rights()
+        table = Table(["name", "description", "default"])
+        for name, right_def in rights.items():
+            desc = right_def.get("desc", "")
+            default = right_def.get("default", "")
+            table.add_row([name, desc, "{}".format(default)])
+        self.write_table(table)
+
+    def do_show_rules(self, arg):
+        rules = self.policy.get_rules()
+        table = Table(["name", "description", "default"])
+        for name, rule_def in rules.items():
+            desc = rule_def.get("desc", "")
+            default = rule_def.get("default", "")
+            table.add_row([name, desc, "{}".format(default)])
+        self.write_table(table)
+
+    def do_show_config(self, arg):
+        config = self.policy.get_config()
+        self.write_string(json.dumps(config, indent=1))
+
+    def do_show_site_rules(self, arg):
+        args = ["show_site_rules"] + self._split_to_args(arg)
+        if len(args) != 2:
+            self.write_string("Usage: {} site_name".format(args[0]))
+            return
+
+        site_name = args[1]
+        rules = self.policy.get_rules()
+        table = Table(["Rule", "Result"])
+        for rule_name, _ in rules.items():
+            result, err = self._eval_rule(site_name, rule_name)
+            if err:
+                self.write_error(err)
+                return
+            else:
+                table.add_row([rule_name, result])
+        self.write_table(table)
+
+    def _eval_right(self, user_name, right_name, site_name):
+        result, err = self.policy.evaluate_user_right_on_site(
+            user_name=user_name, site_name=site_name, right_name=right_name
+        )
+        if err:
+            return result, err
+        if result is None:
+            rights = self.policy.get_rights()
+            right_def = rights[right_name]
+            return "({})".format(right_def.get("default", "?")), ""
+        else:
+            return "{}".format(result), ""
+
+    def do_eval_right(self, arg):
+        args = ["eval_right"] + self._split_to_args(arg)
+        if len(args) != 4:
+            self.write_string("Usage: {} user_name right_name site_name".format(args[0]))
+            return
+
+        user_name = args[1]
+        right_name = args[2]
+        site_name = args[3]
+        result, err = self._eval_right(user_name=user_name, site_name=site_name, right_name=right_name)
+        if err:
+            self.write_error(err)
+        else:
+            self.write_string(result)
+
+    def do_eval_user(self, arg):
+        args = ["eval_user"] + self._split_to_args(arg)
+        if len(args) != 3:
+            self.write_string("Usage: {} user_name site_name".format(args[0]))
+            return
+
+        user_name = args[1]
+        site_name = args[2]
+        table = Table(["Right", "Result"])
+        rights = self.policy.get_rights()
+        for right_name, right_def in rights.items():
+            result, err = self._eval_right(user_name=user_name, site_name=site_name, right_name=right_name)
+            if err:
+                self.write_error(err)
+                return
+            else:
+                table.add_row([right_name, result])
+        self.write_table(table)
+
+    def _eval_rule(self, site_name, rule_name):
+        result, err = self.policy.evaluate_rule_on_site(site_name=site_name, rule_name=rule_name)
+        if err:
+            return result, err
+
+        if result is None:
+            rules = self.policy.get_rules()
+            rule_def = rules[rule_name]
+            return "({})".format(rule_def.get("default", "?")), ""
+        else:
+            return "{}".format(result), ""
+
+    def do_eval_rule(self, arg):
+        args = ["eval_rule"] + self._split_to_args(arg)
+        if len(args) != 3:
+            self.write_string("Usage: {} site_name rule_name".format(args[0]))
+            return
+
+        site_name = args[1]
+        rule_name = args[2]
+        result, err = self._eval_rule(site_name=site_name, rule_name=rule_name)
+        if err:
+            self.write_error(err)
+        else:
+            self.write_string(result)
+
+    def write_string(self, data: str):
+        content = data + "\n"
+        self.stdout.write(content)
+
+    def write_table(self, table: Table):
+        table.write(self.stdout)
+
+    def write_error(self, err: str):
+        content = "Error: " + err + "\n"
+        self.stdout.write(content)
+
+
+def main():
+    """
+    Tool to help preview and see the details of an authorization policy with command line commands.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--policy", "-p", type=str, help="authz policy file", required=False, default="")
+    parser.add_argument("--defs", "-d", type=str, help="authz definition file", required=False, default="")
+    parser.add_argument("--config", "-c", type=str, help="authz config file", required=False, default="")
+
+    args = parser.parse_args()
+
+    if args.policy:
+        with open(args.policy) as file:
+            config = json.load(file)
+            err = validate_policy_config(config)
+            if err:
+                print("Policy config error: {}".format(err))
+                return
+    else:
+        assert args.defs, "missing authz definition file"
+        assert args.config, "missing authz config file"
+        with open(args.defs) as file:
+            defs = json.load(file)
+
+        with open(args.config) as file:
+            config = json.load(file)
+
+        config.update(defs)
+
+    commander = Commander(Policy(config))
+    commander.cmdloop(intro="Type help or ? to list commands.")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/nvflare/fuel/hci/tools/make_pwd.py b/nvflare/fuel/hci/tools/make_pwd.py
new file mode 100644
index 0000000000..27906aa94f
--- /dev/null
+++ b/nvflare/fuel/hci/tools/make_pwd.py
@@ -0,0 +1,44 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import getpass
+
+from nvflare.fuel.hci.security import hash_password
+
+
+def main():
+    """
+    TODO: should this file be removed?
+
+    """
+    user_name = input("User Name: ")
+
+    pwd = getpass.getpass("Password (8 or more chars): ")
+
+    if len(pwd) < 8:
+        print("Invalid password - must have at least 8 chars")
+        return
+
+    pwd2 = getpass.getpass("Confirm Password: ")
+
+    if pwd != pwd2:
+        print("Passwords mismatch")
+        return
+
+    result = hash_password(user_name + pwd)
+    print("Password Hash: {}".format(result))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/nvflare/fuel/hci/zip_utils.py b/nvflare/fuel/hci/zip_utils.py
new file mode 100644
index 0000000000..6625ad44ca
--- /dev/null
+++ b/nvflare/fuel/hci/zip_utils.py
@@ -0,0 +1,111 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import os
+from zipfile import ZipFile
+
+
+def get_all_file_paths(directory):
+    """
+    Get all file paths in the directory.
+    Args:
+        directory: directory to get all paths for
+
+    Returns: all paths in the provided directory
+    """
+    file_paths = []
+
+    # crawling through directory and subdirectories
+    for root, directories, files in os.walk(directory):
+        for filename in files:
+            file_paths.append(os.path.join(root, filename))
+        for dir_name in directories:
+            file_paths.append(os.path.join(root, dir_name))
+
+    return file_paths
+
+
+def _zip_directory(root_dir: str, folder_name: str, writer):
+    """
+    Create a zip archive file for the specified directory.
+
+    Args:
+        root_dir: root path that contains the folder to be zipped
+        folder_name: path to the folder to be zipped, relative to root_dir
+    """
+    dir_name = os.path.join(root_dir, folder_name)
+    assert os.path.exists(dir_name), 'directory "{}" does not exist'.format(dir_name)
+    assert os.path.isdir(dir_name), '"{}" is not a valid directory'.format(dir_name)
+
+    file_paths = get_all_file_paths(dir_name)
+
+    # writing files to a zipfile
+    with ZipFile(writer, "w") as z:
+        # writing each file one by one
+        for full_path in file_paths:
+            rel_path = os.path.relpath(full_path, root_dir)
+            z.write(full_path, arcname=rel_path)
+
+
+def zip_directory_to_file(root_dir: str, folder_name: str, output_file_name: str):
+    """
+    Create a zip archive file for the specified directory.
+
+    Args:
+        root_dir: root path that contains the folder to be zipped
+        folder_name: path to the folder to be zipped, relative to root_dir
+        output_file_name: name of the output file
+    """
+    _zip_directory(root_dir, folder_name, output_file_name)
+
+
+def zip_directory_to_bytes(root_dir: str, folder_name: str):
+    bio = io.BytesIO()
+    _zip_directory(root_dir, folder_name, bio)
+    return bio.getvalue()
+
+
+def _unzip_all(reader, output_dir_name: str):
+    """
+    Decompress a zip archive file and extract all files to the specified output directory.
+
+    Args:
+        reader: the input zip reader
+        output_dir_name: the output directory for extracted content
+    """
+    assert os.path.exists(output_dir_name), 'output directory "{}" does not exist'.format(output_dir_name)
+
+    assert os.path.isdir(output_dir_name), '"{}" is not a valid directory'.format(output_dir_name)
+
+    with ZipFile(reader, "r") as z:
+        z.extractall(output_dir_name)
+
+
+def unzip_all_from_file(zip_file_name: str, output_dir_name: str):
+    """
+    Decompress a zip archive file and extract all files to the specified output directory.
+
+    Args:
+        zip_file_name: the input zip archive file
+        output_dir_name: the output directory for extracted content
+    """
+    assert os.path.exists(zip_file_name), 'input zip file "{}" does not exist'.format(zip_file_name)
+    assert os.path.isfile(zip_file_name), '"{}" is not a valid file'.format(zip_file_name)
+
+    _unzip_all(zip_file_name, output_dir_name)
+
+
+def unzip_all_from_bytes(data, output_dir_name: str):
+    _unzip_all(io.BytesIO(data), output_dir_name)
diff --git a/nvflare/fuel/sec/__init__.py b/nvflare/fuel/sec/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/fuel/sec/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/fuel/sec/audit.py b/nvflare/fuel/sec/audit.py
new file mode 100644
index 0000000000..21f3a18db7
--- /dev/null
+++ b/nvflare/fuel/sec/audit.py
@@ -0,0 +1,82 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import uuid
+from datetime import datetime
+
+
+class Auditor(object):
+    """
+    Manages the audit file to log events.
+    """
+
+    def __init__(self, audit_file_name: str):
+        assert isinstance(audit_file_name, str), "audit_file_name must be str"
+        if os.path.exists(audit_file_name):
+            assert os.path.isfile(audit_file_name), "audit_file_name is not a valid file"
+
+        # create/open the file
+        self.audit_file = open(audit_file_name, "a")
+
+    def add_event(self, user: str, action: str, ref: str = "", msg: str = "") -> str:
+        event_id = uuid.uuid4()
+        event_id_str = "{}".format(event_id)
+
+        if len(ref) > 0:
+            ref = " [R:{}] ".format(ref)
+        else:
+            ref = " "
+
+        if len(msg) > 0:
+            msg = " [M:{}]".format(msg)
+
+        line = "[E:{}]{}[T:{}] [U:{}] [A:{}]{}\n".format(event_id, ref, datetime.now(), user, action, msg)
+        self.audit_file.write(line)
+        self.audit_file.flush()
+        return event_id_str
+
+    def close(self):
+        if self.audit_file is not None:
+            self.audit_file.close()
+            self.audit_file = None
+
+
+class AuditService(object):
+    """
+    Service for interacting with Auditor to add events to log.
+    """
+
+    the_auditor = None
+
+    @staticmethod
+    def initialize(audit_file_name: str):
+        if not AuditService.the_auditor:
+            AuditService.the_auditor = Auditor(audit_file_name)
+        return AuditService.the_auditor
+
+    @staticmethod
+    def get_auditor():
+        return AuditService.the_auditor
+
+    @staticmethod
+    def add_event(user: str, action: str, ref: str = "", msg: str = "") -> str:
+        if not AuditService.the_auditor:
+            return ""
+        return AuditService.the_auditor.add_event(user, action, ref, msg)
+
+    @staticmethod
+    def close():
+        if AuditService.the_auditor:
+            AuditService.the_auditor.close()
diff --git a/nvflare/fuel/sec/authz.py b/nvflare/fuel/sec/authz.py
new file mode 100644
index 0000000000..c9c8094d40
--- /dev/null
+++ b/nvflare/fuel/sec/authz.py
@@ -0,0 +1,451 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from typing import List
+
+
+def _validate_value(type_def: dict, value):
+    value_type = type_def.get("type", "bool")
+    if value_type == "bool":
+        if value not in [True, False]:
+            return False
+    elif not isinstance(value, (int, float)):
+        return False
+    return True
+
+
+def validate_policy_config(config: dict) -> str:
+    """
+    Validates that an authorization policy configuration has the right syntax.
+
+    Args:
+        config: configuration dictionary to validate
+
+    Returns: empty string if there are no errors, else a string describing the error encountered
+
+    """
+    if not isinstance(config, dict):
+        return "policy definition must be a dict"
+
+    # validate rules
+    policy_rules = config.get("rules", None)
+    if not policy_rules:
+        return "missing rules in policy"
+    if not isinstance(policy_rules, dict):
+        return "rules must be dict"
+    for rule_name, rule_def in policy_rules.items():
+        if not isinstance(rule_def, dict):
+            return 'bad rule "{}": must be a dict'.format(rule_name)
+
+        default_value = rule_def.get("default", None)
+        if default_value is None:
+            return 'bad rule "{}": missing default value'.format(rule_name)
+
+        rule_type = rule_def.get("type", "bool")
+        if rule_type not in ["bool", "int"]:
+            return 'bad rule "{}": invalid type "{}"'.format(rule_name, rule_type)
+
+        if not _validate_value(rule_def, default_value):
+            return 'bad rule "{}": invalid default "{}"'.format(rule_name, default_value)
+
+    # validate rights
+    policy_rights = config.get("rights", None)
+    if not policy_rights:
+        return "missing rights in policy"
+    if not isinstance(policy_rights, dict):
+        return "rights must be dict"
+    for right_name, right_def in policy_rights.items():
+        if not isinstance(right_def, dict):
+            return 'bad right "{}": must be a dict'.format(right_name)
+
+        precond = right_def.get("precond", None)
+        if precond is not None and precond not in ["selfOrg"]:
+            return 'bad right "{}": unknown precond "{}"'.format(right_name, precond)
+
+        default_value = right_def.get("default", None)
+        if default_value is None:
+            return 'bad right "{}": missing default value'.format(right_name)
+
+        right_type = right_def.get("type", "bool")
+        if right_type not in ["bool", "int"]:
+            return 'bad right "{}": invalid type "{}"'.format(right_name, right_type)
+
+        if not _validate_value(right_def, default_value):
+            return 'bad right "{}": invalid default "{}"'.format(right_name, default_value)
+
+    # validate roles
+    policy_roles = config.get("roles", {})
+    if policy_roles:
+        if not isinstance(policy_roles, dict):
+            return "roles must be dict"
+
+        for role_name, role_desc in policy_roles.items():
+            if not isinstance(role_name, str):
+                return 'bad role name "{}": must be str'.format(role_name)
+            if not isinstance(role_desc, str):
+                return 'bad role desc "{}": must be str'.format(role_desc)
+
+    # validate groups
+    policy_groups = config.get("groups", {})
+    if policy_groups:
+        if not isinstance(policy_groups, dict):
+            return "groups must be dict"
+
+        for grp_name, grp_def in policy_groups.items():
+            if not isinstance(grp_def, dict):
+                return 'bad group "{}": group def must be dict'.format(grp_name)
+            rules = grp_def.get("rules", None)
+            if rules:
+                if not isinstance(rules, dict):
+                    return 'bad group "{}": rules must be dict'.format(grp_name)
+                for rule_name, rule_value in rules.items():
+                    rule_def = policy_rules.get(rule_name, None)
+                    if not rule_def:
+                        return 'bad group "{}": unknown rule "{}"'.format(grp_name, rule_name)
+                    if not _validate_value(rule_def, rule_value):
+                        return 'bad group "{}": invalid value "{}" for rule "{}"'.format(
+                            grp_name, rule_value, rule_name
+                        )
+
+            role_rights = grp_def.get("role_rights", None)
+            if role_rights:
+                if not isinstance(role_rights, dict):
+                    return 'bad group "{}": role_rights must be dict'.format(grp_name)
+                for role_name, rights in role_rights.items():
+                    if not isinstance(rights, dict):
+                        return 'bad group "{}": rights of role "{}" must be dict'.format(grp_name, role_name)
+                    for right_name, right_value in rights.items():
+                        right_def = policy_rights.get(right_name, None)
+                        if not right_def:
+                            return 'bad group "{}": unknown right "{}" in role "{}" must be dict'.format(
+                                grp_name, right_name, role_name
+                            )
+                        if not _validate_value(right_def, right_value):
+                            return 'bad group "{}": invalid value "{}" for right "{}" in role "{}" must be dict'.format(
+                                grp_name, right_value, right_name, role_name
+                            )
+
+    # validate orgs
+    policy_orgs = config.get("orgs", None)
+    if not policy_orgs:
+        return "missing orgs in policy"
+
+    if not isinstance(policy_orgs, dict):
+        return "orgs must be a dict"
+
+    for org_name, groups in policy_orgs.items():
+        if not isinstance(groups, list):
+            return 'bad org "{}": groups must be a list'.format(org_name)
+        if len(groups) <= 0:
+            return 'bad org "{}": groups not defined'.format(org_name)
+        for grp_name in groups:
+            if not isinstance(grp_name, str):
+                return 'bad org "{}": group name must be str'.format(org_name)
+            grp_def = policy_groups.get(grp_name, None)
+            if not grp_def:
+                return 'bad org "{}": undefined group "{}"'.format(org_name, grp_name)
+
+    # validate users
+    policy_users = config.get("users", None)
+    if not policy_users:
+        return "missing users in policy"
+
+    if not isinstance(policy_users, dict):
+        return "users must be dict"
+
+    for user_name, user_def in policy_users.items():
+        if not isinstance(user_def, dict):
+            return 'bad user "{}": definition must be dict'.format(user_name)
+        org_name = user_def.get("org", None)
+        if not org_name:
+            return 'bad user "{}": missing org'.format(user_name)
+
+        org_def = policy_orgs.get(org_name)
+        if not org_def:
+            return 'bad user "{}": undefined org "{}"'.format(user_name, org_name)
+
+        roles = user_def.get("roles", None)
+        if not roles:
+            return 'bad user "{}": missing roles'.format(user_name)
+        if not isinstance(roles, list):
+            return 'bad user "{}": roles must be list'.format(user_name)
+        if len(roles) <= 0:
+            return 'bad user "{}": no roles defined'.format(user_name)
+        for role_name in roles:
+            if not isinstance(role_name, str):
+                return 'bad user "{}": role name must be str'.format(user_name)
+            role_def = policy_roles.get(role_name, None)
+            if not role_def:
+                return 'bad user "{}": undefined role "{}" must be str'.format(user_name, role_name)
+
+    # validate sites
+    sites = config.get("sites", None)
+    if not sites:
+        return "missing sites in policy"
+
+    if not isinstance(sites, dict):
+        return "sites must be dict"
+
+    for site_name, org_name in sites.items():
+        if org_name not in policy_orgs:
+            return 'bad site "{}": undefined org "{}"'.format(site_name, org_name)
+
+    return ""
+
+
+def _group_rule_key(grp_name: str, rule_name: str):
+    return grp_name + ":" + rule_name
+
+
+def _group_role_right_key(grp_name: str, role_name: str, right_name: str):
+    return grp_name + ":" + role_name + ":" + right_name
+
+
+def _eval_bool(space: dict, keys: [str]):
+    exit_value = None
+    for k in keys:
+        value = space.get(k, None)
+        if value:
+            return True
+        if value is not None:
+            exit_value = False
+    return exit_value
+
+
+def _eval_int(space: dict, keys: [str]):
+    exit_value = None
+    for k in keys:
+        value = space.get(k, None)
+        if value is not None:
+            if exit_value is None or exit_value < value:
+                exit_value = value
+    return exit_value
+
+
+class Policy(object):
+    """
+    Authorization policy definition with methods to access information about the policy. Init creates the internal
+    representation of the policy from a config dictionary.
+
+    Policy evaluation result:
+
+    For bool type of rules or rights:
+
+        True - the rule is satisfied or the right is granted
+        False - the rule is not satisfied; the right iis not granted
+        None - the rule or right is not applicable (precondition not met)
+
+    For int type or rules or rights:
+
+        Number - the value of the evaluation
+        None - the rule or right is not applicable (precondition not met)
+    """
+
+    def __init__(self, conf: dict):
+        self.config = conf
+        self.preconf_valuators = {"selfOrg": self._eval_precond_self_org}
+
+        # compute the rule and right spaces
+        self.rule_space = {}
+        self.right_space = {}
+        groups = conf["groups"]
+        for grp_name, grp_def in groups.items():
+            rules = grp_def.get("rules", None)
+            if rules:
+                for rule_name, rule_value in rules.items():
+                    key = _group_rule_key(grp_name, rule_name)
+                    self.rule_space[key] = rule_value
+
+            role_rights = grp_def.get("role_rights", None)
+            if role_rights:
+                for role_name, rights in role_rights.items():
+                    for right_name, right_value in rights.items():
+                        key = _group_role_right_key(grp_name, role_name, right_name)
+                        self.right_space[key] = right_value
+
+    def get_config(self):
+        return self.config
+
+    def _eval_precond(self, precond: str, user_name: str, org_name: str):
+        evaluator = self.preconf_valuators.get(precond, None)
+        if not evaluator:
+            return None
+        else:
+            return evaluator(user_name, org_name)
+
+    def _eval_precond_self_org(self, user_name: str, org_name: str):
+        users = self.config["users"]
+        user = users[user_name]
+        return user["org"] == org_name
+
+    def _get_org_groups(self, org_name: str):
+        orgs = self.config["orgs"]
+        return orgs.get(org_name, None)
+
+    def evaluate_rule_on_org(self, rule_name: str, org_name: str):
+        rules = self.config["rules"]
+        rule_def = rules.get(rule_name, None)
+        if not rule_def:
+            return None, 'undefined rule "{}"'.format(rule_name)
+
+        rule_type = rule_def.get("type", "bool")
+        groups = self._get_org_groups(org_name)
+        if not groups:
+            return None, 'unknown org "{}"'.format(org_name)
+
+        keys = []
+        for grp_name in groups:
+            keys.append(_group_rule_key(grp_name, rule_name))
+
+        if rule_type == "bool":
+            result = _eval_bool(space=self.rule_space, keys=keys)
+        else:
+            result = _eval_bool(space=self.rule_space, keys=keys)
+
+        if result is None:
+            result = rule_def["default"]
+        return result, ""
+
+    def _get_org_of_site(self, site_name):
+        sites = self.config["sites"]
+        return sites.get(site_name, None)
+
+    def evaluate_rule_on_site(self, rule_name: str, site_name: str):
+        org_name = self._get_org_of_site(site_name)
+        if not org_name:
+            return None, 'unknown site "{}"'.format(site_name)
+        return self.evaluate_rule_on_org(rule_name, org_name)
+
+    def evaluate_user_right_on_org(self, right_name: str, user_name: str, org_name: str):
+        rights = self.config["rights"]
+        right_def = rights.get(right_name, None)
+        if not right_def:
+            return None, 'undefined right "{}"'.format(right_name)
+
+        right_type = right_def.get("type", "bool")
+        groups = self._get_org_groups(org_name)
+        if not groups:
+            return None, 'unknown org "{}"'.format(org_name)
+
+        users = self.config["users"]
+        user = users.get(user_name, None)
+        if not user:
+            return None, 'unknown user "{}"'.format(user_name)
+
+        precond = right_def.get("precond", None)
+        if precond:
+            matched = self._eval_precond(precond, user_name, org_name)
+            if not matched:
+                if right_type == "bool":
+                    return False, ""
+                else:
+                    return 0, ""
+
+        roles = user["roles"]
+        keys = []
+        for grp_name in groups:
+            for role_name in roles:
+                keys.append(_group_role_right_key(grp_name, role_name, right_name))
+
+        if right_type == "bool":
+            result = _eval_bool(self.right_space, keys)
+        else:
+            result = _eval_int(self.right_space, keys)
+
+        if result is None:
+            result = right_def["default"]
+        return result, ""
+
+    def evaluate_user_right_on_site(self, right_name: str, user_name: str, site_name: str):
+        org_name = self._get_org_of_site(site_name)
+        if not org_name:
+            return None, 'unknown site "{}"'.format(site_name)
+        return self.evaluate_user_right_on_org(right_name, user_name, org_name)
+
+    def get_user(self, user_name: str):
+        users = self.config["users"]
+        return users.get(user_name, None)
+
+    def get_users(self):
+        return self.config["users"]
+
+    def get_sites(self):
+        return self.config["sites"]
+
+    def get_rights(self):
+        return self.config["rights"]
+
+    def get_rules(self):
+        return self.config["rules"]
+
+    def get_right_type(self, right_name: str):
+        rights = self.config["rights"]
+        right_def = rights.get(right_name, None)
+        if not right_def:
+            return None
+        return right_def.get("type", "bool")
+
+
+class AuthzContext(object):
+    """
+    Base class to contain context data for authorization.
+    """
+
+    def __init__(self, user_name: str, site_names: List[str]):
+        self.user_name = user_name
+        self.site_names = site_names
+        self.attrs = {}
+
+    def set_attr(self, key: str, value):
+        self.attrs[key] = value
+
+    def get_attr(self, key: str, default=None):
+        return self.attrs.get(key, default)
+
+
+class Authorizer(object):
+    """
+    Base class containing the authorization policy.
+    """
+
+    def __init__(self):
+        self.policy = None
+        self.last_load_time = None
+
+    def get_policy(self) -> Policy:
+        return self.policy
+
+    def authorize(self, ctx: AuthzContext) -> (object, str):
+        return True, ""
+
+    def evaluate_user_right_on_site(self, right_name: str, user_name: str, site_name: str):
+        if not self.policy:
+            return None, "policy not defined"
+        return self.policy.evaluate_user_right_on_site(right_name=right_name, user_name=user_name, site_name=site_name)
+
+    def evaluate_rule_on_site(self, rule_name: str, site_name: str):
+        if not self.policy:
+            return None, "policy not defined"
+
+        return self.policy.evaluate_rule_on_site(rule_name=rule_name, site_name=site_name)
+
+    def load_policy(self, policy_config: dict) -> str:
+        err = validate_policy_config(policy_config)
+        if err:
+            return err
+
+        self.policy = Policy(policy_config)
+        self.last_load_time = time.time()
+        return ""
diff --git a/nvflare/fuel/sec/security_content_service.py b/nvflare/fuel/sec/security_content_service.py
new file mode 100644
index 0000000000..e4cc51479a
--- /dev/null
+++ b/nvflare/fuel/sec/security_content_service.py
@@ -0,0 +1,128 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import pickle
+from enum import Enum
+
+from cryptography import x509
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import padding
+
+
+class LoadResult(Enum):
+    """
+    Constants for different results when loading secure content.
+    """
+
+    OK = "ok"
+    NOT_MANAGED = "notManaged"
+    NO_SUCH_CONTENT = "noSuchContent"
+    NOT_SIGNED = "notSigned"
+    INVALID_SIGNATURE = "invalidSignature"
+    INVALID_CONTENT = "invalidContent"
+
+
+class SecurityContentManager(object):
+    """
+    Content manager used by SecurityContentService to load secure content.
+    """
+
+    def __init__(self, content_folder, signature_filename="signature.pkl", root_cert="rootCA.pem"):
+        self.content_folder = content_folder
+        signature_path = os.path.join(self.content_folder, signature_filename)
+        rootCA_cert_path = os.path.join(self.content_folder, root_cert)
+        if os.path.exists(signature_path) and os.path.exists(rootCA_cert_path):
+            self.signature = pickle.load(open(signature_path, "rb"))
+            cert = x509.load_pem_x509_certificate(open(rootCA_cert_path, "rb").read(), default_backend())
+            self.public_key = cert.public_key()
+            self.valid_config = True
+        else:
+            self.signature = dict()
+            self.valid_config = False
+
+    def load_content(self, file_under_verification):
+        """Loads the data of the file under verification and verifies that the signature is valid.
+
+        Args:
+            file_under_verification: file to load and verify
+
+        Returns: Tuple of the file data and the LoadResult. File data may be None if the data cannot be loaded.
+
+        """
+        full_path = os.path.join(self.content_folder, file_under_verification)
+        data = None
+        if not os.path.exists(full_path):
+            return data, LoadResult.NO_SUCH_CONTENT
+
+        with open(full_path, "rb") as f:
+            data = f.read()
+        if not data:
+            return data, LoadResult.NO_SUCH_CONTENT
+
+        if self.valid_config and file_under_verification in self.signature:
+            signature = self.signature[file_under_verification]
+            try:
+                self.public_key.verify(
+                    signature=signature,
+                    data=data,
+                    padding=padding.PSS(mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.MAX_LENGTH),
+                    algorithm=hashes.SHA256(),
+                )
+                result = LoadResult.OK
+            except InvalidSignature:
+                result = LoadResult.INVALID_SIGNATURE
+        else:
+            result = LoadResult.NOT_SIGNED
+        return data, result
+
+
+class SecurityContentService(object):
+    """
+    Uses SecurityContentManager to load secure content.
+    """
+
+    security_content_manager = None
+
+    @staticmethod
+    def initialize(content_folder: str, signature_filename="signature.pkl", root_cert="rootCA.pem"):
+        if SecurityContentService.security_content_manager is None:
+            SecurityContentService.security_content_manager = SecurityContentManager(
+                content_folder, signature_filename, root_cert
+            )
+
+    @staticmethod
+    def load_content(file_under_verification):
+        if not SecurityContentService.security_content_manager:
+            return None, LoadResult.NOT_MANAGED
+
+        return SecurityContentService.security_content_manager.load_content(file_under_verification)
+
+    @staticmethod
+    def load_json(file_under_verification):
+        json_data = None
+
+        data_bytes, result = SecurityContentService.security_content_manager.load_content(file_under_verification)
+
+        if data_bytes:
+            try:
+                data_text = data_bytes.decode("ascii")
+                json_data = json.loads(data_text)
+            except json.JSONDecodeError:
+                return None, LoadResult.INVALID_CONTENT
+
+        return json_data, result
diff --git a/nvflare/fuel/utils/__init__.py b/nvflare/fuel/utils/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/fuel/utils/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/fuel/utils/argument_utils.py b/nvflare/fuel/utils/argument_utils.py
new file mode 100644
index 0000000000..740852ddf8
--- /dev/null
+++ b/nvflare/fuel/utils/argument_utils.py
@@ -0,0 +1,64 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distutils.util import strtobool
+
+
+def parse_var(s):
+    """Returns (key, value) tuple from string with equals sign with the portion before the first equals sign as the key
+    and the rest as the value.
+
+    Args:
+        s: string to parse
+
+    Returns: Tuple of key and value
+    """
+    items = s.split("=")
+    key = items[0].strip()  # we remove blanks around keys, as is logical
+    value = ""
+    if len(items) > 1:
+        # rejoin the rest:
+        value = "=".join(items[1:])
+    return key, value
+
+
+def parse_vars(items):
+    """Converts a list of key value pairs into a dictionary.
+
+    Args:
+        items: list like ['a=1', 'b=2', 'c=3']
+
+    Returns: dictionary like {'a': '1', 'b': '2', 'c': '3'}
+
+    """
+    d = {}
+    if items:
+        for item in items:
+            key, value = parse_var(item)
+
+            # d[key] = value
+            try:
+                d[key] = int(value)
+            except ValueError:
+                pass
+                try:
+                    d[key] = float(value)
+                except ValueError:
+                    pass
+                    try:
+                        d[key] = bool(strtobool(str(value)))
+                    except ValueError:
+                        pass
+                        d[key] = value
+    return d
diff --git a/nvflare/fuel/utils/class_utils.py b/nvflare/fuel/utils/class_utils.py
new file mode 100644
index 0000000000..abd99bd41e
--- /dev/null
+++ b/nvflare/fuel/utils/class_utils.py
@@ -0,0 +1,103 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import inspect
+import pkgutil
+
+
+def get_class(class_path):
+    module_name, class_name = class_path.rsplit(".", 1)
+
+    try:
+        module_ = importlib.import_module(module_name)
+
+        try:
+            class_ = getattr(module_, class_name)
+        except AttributeError:
+            raise ValueError("Class {} does not exist".format(class_path))
+    except AttributeError:
+        raise ValueError("Module {} does not exist".format(class_path))
+
+    return class_
+
+
+def instantiate_class(class_path, init_params):
+    """
+    Method for creating an instance for the class.
+
+    Args:
+        class_path: full path of the class
+        init_params: A dictionary that contains the name of the transform and constructor input
+        arguments. The transform name will be appended to `medical.common.transforms` to make a
+        full name of the transform to be built.
+    """
+
+    c = get_class(class_path)
+    try:
+        if init_params:
+            instance = c(**init_params)
+        else:
+            instance = c()
+    except TypeError as e:
+        raise ValueError("Class {} has parameters error.".format(class_path), str(e))
+
+    return instance
+
+
+def get_object_method(obj, method_name):
+    op = getattr(obj, method_name, None)
+    if op is None or not callable(op):
+        return None
+    return op
+
+
+def get_instance_method(instance, method_name):
+    return get_object_method(instance, method_name)
+
+
+def get_config_classname(config_dict: dict):
+    class_name = config_dict.get("name", None)
+    if not class_name:
+        class_name = config_dict.get("path", "")
+    return class_name
+
+
+class ModuleScanner:
+    def __init__(self, base_pkgs: [str], module_names: [str], exclude_libs=True):
+        self.base_pkgs = base_pkgs
+        self.module_names = module_names
+        self.exclude_libs = exclude_libs
+        self._class_table = {}
+        self._create_classes_table()
+
+    def _create_classes_table(self):
+        for base in self.base_pkgs:
+            package = __import__(base)
+
+            for importer, modname, ispkg in pkgutil.walk_packages(path=package.__path__, prefix=package.__name__ + "."):
+
+                if modname.startswith(base):
+                    if not self.exclude_libs or (".libs" not in modname):
+                        if any(name in modname for name in self.module_names):
+                            try:
+                                module = importlib.import_module(modname)
+                                for name, obj in inspect.getmembers(module):
+                                    if inspect.isclass(obj) and obj.__module__ == modname:
+                                        self._class_table[name] = modname
+                            except ModuleNotFoundError as ex:
+                                pass
+
+    def get_module_name(self, class_name):
+        return self._class_table.get(class_name, None)
diff --git a/nvflare/fuel/utils/comm.py b/nvflare/fuel/utils/comm.py
new file mode 100644
index 0000000000..65e7f6ca72
--- /dev/null
+++ b/nvflare/fuel/utils/comm.py
@@ -0,0 +1,86 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+
+
+class Queue(object):
+    def __init__(self, name):
+        self.name = name
+        self.items = []
+        self._update_lock = threading.Lock()
+        self.next_seq_no = 1
+
+    def append(self, item):
+        with self._update_lock:
+            self.items.append(item)
+            seq_no = self.next_seq_no
+            self.next_seq_no += 1
+            return seq_no
+
+    def len(self):
+        with self._update_lock:
+            return len(self.items)
+
+    def get(self, default=None):
+        with self._update_lock:
+            if len(self.items) > 0:
+                result = self.items[0]
+            else:
+                result = default
+        return result
+
+    def pop(self, default=None):
+        with self._update_lock:
+            if len(self.items) > 0:
+                result = self.items.pop(0)
+            else:
+                result = default
+        return result
+
+    def fill(self, data, limit):
+        with self._update_lock:
+            count = limit - len(self.items)
+            if count > 0:
+                for _ in range(count):
+                    self.items.append(data)
+
+    def clear(self):
+        with self._update_lock:
+            self.items = []
+
+
+class Channel(object):
+    def __init__(self, src, dest):
+        self.src = src
+        self.dest = dest
+        self.req = Queue("req")
+        self.reply = Queue("reply")
+
+    def send(self, src, data):
+        if src == self.src:
+            self.req.append(data)
+        elif src == self.dest:
+            self.reply.append(data)
+
+    def receive(self, src, default=None):
+        if src == self.src:
+            return self.reply.pop(default=default)
+        elif src == self.dest:
+            return self.req.pop(default=default)
+        else:
+            return default
+
+    def __str__(self):
+        return "Channel({}<=>{})".format(self.src.name, self.dest.name)
diff --git a/nvflare/fuel/utils/component_builder.py b/nvflare/fuel/utils/component_builder.py
new file mode 100644
index 0000000000..654a918a5d
--- /dev/null
+++ b/nvflare/fuel/utils/component_builder.py
@@ -0,0 +1,88 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import abstractmethod
+
+from nvflare.fuel.common.excepts import ConfigError
+from nvflare.fuel.utils.class_utils import instantiate_class
+
+
+class ComponentBuilder:
+    @abstractmethod
+    def get_module_scanner(self):
+        """
+        Provide the package module scanner.
+        Returns: module_scanner
+
+        """
+        pass
+
+    def build_component(self, config_dict):
+        if not config_dict:
+            return None
+
+        if not isinstance(config_dict, dict):
+            raise ConfigError("component config must be dict")
+
+        if config_dict.get("disabled") is True:
+            return None
+
+        class_args = config_dict.get("args", dict())
+        for k, v in class_args.items():
+            if isinstance(v, dict):
+                # try to replace the arg with a component
+                try:
+                    t = self.build_component(v)
+                    class_args[k] = t
+                except BaseException:
+                    pass
+        class_path = self.get_class_path(config_dict)
+
+        # Handle the special case, if config pass in the class_attributes, use the user defined class attributes
+        # parameters directly.
+        if "class_attributes" in class_args:
+            class_args = class_args["class_attributes"]
+
+        return instantiate_class(class_path, class_args)
+
+    def get_class_path(self, config_dict):
+        if "path" in config_dict.keys():
+            path_spec = config_dict["path"]
+            if not isinstance(path_spec, str):
+                raise ConfigError("path spec must be str")
+
+            if len(path_spec) <= 0:
+                raise ConfigError("path spec must not be empty")
+
+            class_path = format(path_spec)
+            parts = class_path.split(".")
+            if len(parts) < 2:
+                raise ConfigError("invalid class path '{}': missing module name".format(class_path))
+        else:
+            if "name" not in config_dict:
+                raise ConfigError("class name or path must be specified")
+
+            class_name = config_dict["name"]
+
+            if not isinstance(class_name, str):
+                raise ConfigError("class name must be str")
+
+            if len(class_name) <= 0:
+                raise ConfigError("class name must not be empty")
+            module_name = self.get_module_scanner().get_module_name(class_name)
+            if module_name is None:
+                raise ConfigError('Cannot find component class "{}"'.format(class_name))
+            class_path = module_name + ".{}".format(class_name)
+
+        return class_path
diff --git a/nvflare/fuel/utils/dict_utils.py b/nvflare/fuel/utils/dict_utils.py
new file mode 100644
index 0000000000..58626b7dc7
--- /dev/null
+++ b/nvflare/fuel/utils/dict_utils.py
@@ -0,0 +1,56 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import json
+
+
+def update(d, u):
+    for k, v in u.items():
+        if isinstance(v, collections.Mapping):
+            d[k] = update(d.get(k, {}), v)
+        else:
+            d[k] = v
+    return d
+
+
+def update_configs_with_envs(configs, env):
+    for k, v in configs.items():
+        if isinstance(v, list):
+            length = len(v)
+            for i in range(length):
+                if isinstance(v[i], dict):
+                    configs[k][i] = update_configs_with_envs(v[i], env)
+        elif isinstance(v, dict):
+            configs[k] = update_configs_with_envs(v, env)
+        elif isinstance(v, str):
+            configs[k] = v.format(**env)
+    return configs
+
+
+def merge_dict(dict1, dict2):
+    return {**dict1, **dict2}
+
+
+def extract_first_level_primitive(d):
+    result = {}
+    for k, v in d.items():
+        if type(v) in (int, float, bool, str):
+            result[k] = v
+    return result
+
+
+def save_to_json(data, path, sort_keys=False, indent=None):
+    with open(path, "w") as f:
+        json.dump(data, f, sort_keys=sort_keys, indent=indent)
diff --git a/nvflare/fuel/utils/json_scanner.py b/nvflare/fuel/utils/json_scanner.py
new file mode 100644
index 0000000000..2f53dbca2e
--- /dev/null
+++ b/nvflare/fuel/utils/json_scanner.py
@@ -0,0 +1,129 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+
+from nvflare.fuel.common.excepts import ConfigError
+
+
+class Node(object):
+    """
+    A JSON element with additional data.
+    """
+
+    def __init__(self, element):
+        self.parent = None
+        self.element = element
+        self.level = 0
+        self.key = ""
+        self.position = 0
+        self.paths = []
+        self.processor = None
+        self.exit_cb = None  # node_exit_cb_signature(node: Node)
+        self.props = {}
+
+    def path(self):
+        if len(self.paths) <= 0:
+            return ""
+
+        return ".".join(self.paths)
+
+    def parent_element(self):
+        if self.parent:
+            return self.parent.element
+        else:
+            return None
+
+
+def _child_node(node: Node, key, pos, element) -> Node:
+    child = Node(element)
+    child.processor = node.processor
+    child.level = node.level + 1
+    child.position = pos
+    child.parent = node
+    child.paths = copy.copy(node.paths)
+
+    child.key = key
+    if pos > 0:
+        child.key = "#{}".format(pos)
+
+    child.paths.append(child.key)
+    return child
+
+
+class JsonObjectProcessor(object):
+    """
+    JsonObjectProcessor is used to process JSON elements by the scan_json() function.
+
+    """
+
+    def process_element(self, node: Node):
+        """
+        This method is called by the scan() function for each JSON element scanned.
+
+        Args:
+            node: the node representing the JSON element
+
+        Returns:
+
+        """
+        pass
+
+
+class JsonScanner(object):
+    """
+    Scanner for processing JSON data.
+    """
+
+    def __init__(self, json_data: dict, location=None):
+        assert isinstance(json_data, dict), "json_data must be dict"
+        self.location = location
+        self.data = json_data
+
+    def _do_scan(self, node: Node):
+        try:
+            node.processor.process_element(node)
+        except BaseException as ex:
+            if self.location:
+                raise ConfigError("Error processing {} in JSON element {}: {}".format(self.location, node.path(), ex))
+            else:
+                raise ConfigError("Error in JSON element {}: {}".format(node.path(), ex))
+
+        element = node.element
+
+        if isinstance(element, dict):
+            # need to make a copy of the element dict in case the processor modifies the dict
+            iter_dict = copy.copy(element)
+            for k, v in iter_dict.items():
+                self._do_scan(_child_node(node, k, 0, v))
+        elif isinstance(element, list):
+            for i in range(len(element)):
+                self._do_scan(_child_node(node, node.key, i + 1, element[i]))
+
+        if node.exit_cb is not None:
+            try:
+                node.exit_cb(node)
+            except BaseException as ex:
+                if self.location:
+                    raise ConfigError(
+                        "Error post-processing {} in JSON element {}: {}".format(self.location, node.path(), ex)
+                    )
+                else:
+                    raise ConfigError("Error post-processing JSON element {}: {}".format(node.path(), ex))
+
+    def scan(self, processor: JsonObjectProcessor):
+        assert isinstance(processor, JsonObjectProcessor), "processor must be JsonObjectProcessor"
+        node = Node(self.data)
+        node.processor = processor
+        self._do_scan(node)
diff --git a/nvflare/fuel/utils/pipe/__init__.py b/nvflare/fuel/utils/pipe/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/fuel/utils/pipe/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/fuel/utils/pipe/file_pipe.py b/nvflare/fuel/utils/pipe/file_pipe.py
new file mode 100644
index 0000000000..5dadc18dd9
--- /dev/null
+++ b/nvflare/fuel/utils/pipe/file_pipe.py
@@ -0,0 +1,130 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import uuid
+
+from .pipe import Pipe
+
+
+class FilePipe(Pipe):
+    """
+    Implementation of communication through the file system.
+    """
+
+    def __init__(self, root_path: str, name: str):
+        assert os.path.exists(root_path), 'root path "{}" does not exist'.format(root_path)
+        pipe_path = os.path.join(root_path, name)
+
+        if not os.path.exists(pipe_path):
+            os.mkdir(pipe_path)
+
+        x_path = os.path.join(pipe_path, "x")
+        if not os.path.exists(x_path):
+            os.mkdir(x_path)
+            print("created {}".format(x_path))
+
+        y_path = os.path.join(pipe_path, "y")
+        if not os.path.exists(y_path):
+            os.mkdir(y_path)
+            print("created {}".format(y_path))
+
+        t_path = os.path.join(pipe_path, "t")
+        if not os.path.exists(t_path):
+            os.mkdir(t_path)
+            print("created {}".format(t_path))
+
+        Pipe.__init__(self, name)
+        self.pipe_path = pipe_path
+        self.x_path = x_path
+        self.y_path = y_path
+        self.t_path = t_path
+
+    def _clear_dir(self, p: str):
+        file_list = os.listdir(p)
+        if file_list:
+            for f in file_list:
+                os.remove(os.path.join(p, f))
+
+    def _topic_to_file_name(self, topic: str):
+        return "{}.{}".format(topic, uuid.uuid4())
+
+    def _file_name_to_topic(self, file_name: str):
+        parts = file_name.split(".")
+        return ".".join(parts[0 : len(parts) - 1])
+
+    def _create_file(self, dir: str, topic: str, data_bytes):
+        file_name = self._topic_to_file_name(topic)
+        file_path = os.path.join(dir, file_name)
+
+        tmp_name = file_name + ".tmp"
+        tmp_path = os.path.join(self.t_path, tmp_name)
+        with open(tmp_path, "wb") as f:
+            f.write(data_bytes)
+        os.rename(tmp_path, file_path)
+
+    def clear(self):
+        self._clear_dir(self.x_path)
+        self._clear_dir(self.y_path)
+        self._clear_dir(self.t_path)
+
+    def x_put(self, topic: str, data_bytes):
+        # put it in Y's queue
+        self._create_file(self.y_path, topic, data_bytes)
+
+    def _get_next(self, from_dir: str):
+        # print('get from dir: {}'.format(from_dir))
+        files = os.listdir(from_dir)
+        if files:
+            files = [os.path.join(from_dir, f) for f in files]
+            files.sort(key=os.path.getmtime, reverse=False)
+            next = files[0]
+            # print('got file {}'.format(next))
+
+            topic = self._file_name_to_topic(os.path.basename(next))
+            with open(next, mode="rb") as file:  # b is important -> binary
+                data = file.read()
+            os.remove(next)  # remove this file
+            return topic, data
+        else:
+            return None, None
+
+    def _get_from_dir(self, from_dir: str, timeout=None):
+        if not timeout or timeout <= 0:
+            return self._get_next(from_dir)
+
+        start = time.time()
+        while True:
+            topic, data = self._get_next(from_dir)
+            if topic is not None:
+                return topic, data
+
+            if time.time() - start >= timeout:
+                break
+            time.sleep(2)
+
+        return None, None
+
+    def x_get(self, timeout=None) -> (str, bytes):
+        # read from X's queue
+        return self._get_from_dir(self.x_path, timeout)
+
+    def y_put(self, topic: str, data_bytes):
+        # put it in X's queue
+        self._create_file(self.x_path, topic, data_bytes)
+
+    def y_get(self, timeout=None):
+        # read from Y's queue
+        return self._get_from_dir(self.y_path, timeout)
diff --git a/nvflare/fuel/utils/pipe/pipe.py b/nvflare/fuel/utils/pipe/pipe.py
new file mode 100644
index 0000000000..c0edc7099f
--- /dev/null
+++ b/nvflare/fuel/utils/pipe/pipe.py
@@ -0,0 +1,55 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class EndPoint(object):
+    """
+    Object with put and get functions.
+    """
+
+    def __init__(self, get_func, put_func):
+        self.get_func = get_func
+        self.put_func = put_func
+
+    def put(self, topic: str, data_bytes):
+        self.put_func(topic=topic, data_bytes=data_bytes)
+
+    def get(self, timeout=None):
+        return self.get_func(timeout=timeout)
+
+
+class Pipe(object):
+    """
+    Base class for communication.
+    """
+
+    def __init__(self, name):
+        self.name = name
+        self.x = EndPoint(self.x_get, self.x_put)
+        self.y = EndPoint(self.y_get, self.y_put)
+
+    def clear(self):
+        pass
+
+    def x_put(self, topic: str, data_bytes):
+        pass
+
+    def x_get(self, timeout=None):
+        pass
+
+    def y_put(self, topic: str, data_bytes):
+        pass
+
+    def y_get(self, timeout=None):
+        pass
diff --git a/nvflare/fuel/utils/time_utils.py b/nvflare/fuel/utils/time_utils.py
new file mode 100644
index 0000000000..4ae027d5ae
--- /dev/null
+++ b/nvflare/fuel/utils/time_utils.py
@@ -0,0 +1,30 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+
+def time_to_string(t):
+    """Convert time into a formatted string.
+
+    Args:
+        t: input time string in seconds since the Epoch
+
+    Returns: formatted time string
+
+    """
+    if t is None:
+        return "N/A"
+
+    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t))
diff --git a/nvflare/fuel/utils/wfconf.py b/nvflare/fuel/utils/wfconf.py
new file mode 100644
index 0000000000..3a5bc461da
--- /dev/null
+++ b/nvflare/fuel/utils/wfconf.py
@@ -0,0 +1,289 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import json
+import os
+import re
+
+from nvflare.fuel.common.excepts import ConfigError
+
+from .class_utils import ModuleScanner, get_class, instantiate_class
+from .dict_utils import extract_first_level_primitive, merge_dict
+from .json_scanner import JsonObjectProcessor, JsonScanner, Node
+
+
+class ConfigContext(object):
+    """
+    Object containing configuration context.
+    """
+
+    def __init__(self):
+        self.app_root = ""
+        self.vars = None
+        self.config_json = None
+        self.pass_num = 0
+
+
+class _EnvUpdater(JsonObjectProcessor):
+    def __init__(self, vs, element_filter=None):
+        JsonObjectProcessor.__init__(self)
+        self.vars = vs
+        if element_filter is not None and not callable(element_filter):
+            raise ValueError("element_filter must be a callable function")
+        self.element_filter = element_filter
+
+    def process_element(self, node: Node):
+        element = node.element
+        if isinstance(element, str):
+            if self.element_filter is not None and not self.element_filter(element):
+                return
+            element = self.substitute(element)
+            parent_element = node.parent_element()
+            if node.position > 0:
+                # parent is a list
+                parent_element[node.position - 1] = element
+            else:
+                # parent is a dict
+                parent_element[node.key] = element
+
+    def substitute(self, element: str):
+        a = re.split("{|}", element)
+        if len(a) == 3 and a[0] == "" and a[2] == "":
+            element = self.vars.get(a[1], None)
+        else:
+            element = element.format(**self.vars)
+        return element
+
+
+class Configurator(JsonObjectProcessor):
+    """
+    Base class of Configurator to parse JSON configuration.
+    """
+
+    def __init__(
+        self,
+        app_root: str,
+        cmd_vars: dict,
+        env_config: dict,
+        wf_config_file_name: str,
+        base_pkgs: [str],
+        module_names: [str],
+        exclude_libs=True,
+        default_vars=None,
+        num_passes=1,
+        element_filter=None,
+        var_processor=None,
+    ):
+        JsonObjectProcessor.__init__(self)
+
+        assert isinstance(app_root, str), "app_root must be str"
+
+        assert isinstance(num_passes, int), "num_passes must be int"
+        assert num_passes > 0, "num_passes must > 0"
+
+        if cmd_vars:
+            assert isinstance(cmd_vars, dict), "cmd_vars must be dict"
+
+        if env_config:
+            assert isinstance(env_config, dict), "env_config must be dict"
+
+        assert isinstance(wf_config_file_name, str), "wf_config_file_name must be str"
+        assert os.path.isfile(wf_config_file_name), "wf_config_file_name {} is not a valid file".format(
+            wf_config_file_name
+        )
+        assert os.path.exists(wf_config_file_name), "wf_config_file_name {} does not exist".format(wf_config_file_name)
+
+        if default_vars is not None:
+            assert isinstance(default_vars, dict), "default_vars must be dict"
+        else:
+            default_vars = {}
+
+        self.cmd_vars = cmd_vars
+        self.default_vars = default_vars
+        self.app_root = app_root
+        self.env_config = env_config
+        self.wf_config_file_name = wf_config_file_name
+        self.num_passes = num_passes
+        self.element_filter = element_filter
+
+        self.module_scanner = ModuleScanner(base_pkgs, module_names, exclude_libs)
+        self.all_vars = None
+        self.vars_from_cmd = None
+        self.vars_from_env_config = None
+        self.vars_from_wf_config = None
+        self.config_ctx = None
+        self.var_processor = var_processor
+
+        with open(wf_config_file_name) as file:
+            self.wf_config_data = json.load(file)
+
+        self.json_scanner = JsonScanner(self.wf_config_data, wf_config_file_name)
+
+    def _do_configure(self):
+        vars_from_cmd = {}
+        if self.cmd_vars:
+            vars_from_cmd = copy.copy(self.cmd_vars)
+            for key, value in vars_from_cmd.items():
+                if key.startswith("APP_") and value != "":
+                    vars_from_cmd[key] = os.path.join(self.app_root, value)
+
+        vars_from_env_config = {}
+        if self.env_config:
+            vars_from_env_config = copy.copy(self.env_config)
+            for key, value in vars_from_env_config.items():
+                if key.startswith("APP_") and value != "":
+                    vars_from_env_config[key] = os.path.join(self.app_root, value)
+
+        vars_from_wf_conf = extract_first_level_primitive(self.wf_config_data)
+
+        if "determinism" in self.wf_config_data:
+            vars_from_wf_conf["determinism"] = self.wf_config_data["determinism"]
+
+        # precedence of vars (high to low):
+        #   vars_from_cmd, vars_from_config, vars_from_wf_conf
+        # func merge_dict(d1, d2) gives d2 higher precedence for the same key
+        all_vars = merge_dict(self.default_vars, vars_from_wf_conf)
+        all_vars = merge_dict(all_vars, vars_from_env_config)
+        all_vars = merge_dict(all_vars, vars_from_cmd)
+
+        # update the wf_config with vars
+        self.all_vars = all_vars
+        self.vars_from_cmd = vars_from_cmd
+        self.vars_from_env_config = vars_from_env_config
+        self.vars_from_wf_config = vars_from_wf_conf
+
+        if self.var_processor:
+            self.var_processor.process(self.all_vars, app_root=self.app_root)
+
+        self.json_scanner.scan(_EnvUpdater(all_vars, self.element_filter))
+
+        config_ctx = ConfigContext()
+        config_ctx.vars = self.all_vars
+        config_ctx.app_root = self.app_root
+        config_ctx.config_json = self.wf_config_data
+        self.config_ctx = config_ctx
+
+        self.start_config(self.config_ctx)
+
+        # scan the wf_config again to create components
+        for i in range(self.num_passes):
+            self.config_ctx.pass_num = i + 1
+            self.json_scanner.scan(self)
+
+        # finalize configuration
+        self.finalize_config(self.config_ctx)
+
+    def configure(self):
+        try:
+            self._do_configure()
+        except ConfigError as ex:
+            raise ConfigError("Config error in {}: {}".format(self.wf_config_file_name, ex))
+        except Exception as ex:
+            print("Error processing config {}: {}".format(self.wf_config_file_name, ex))
+            raise ex
+
+    def process_element(self, node: Node):
+        self.process_config_element(self.config_ctx, node)
+
+    def process_args(self, args: dict):
+        return args
+
+    def build_component(self, config_dict):
+        if not config_dict:
+            return None
+
+        if not isinstance(config_dict, dict):
+            raise ConfigError("component config must be dict")
+
+        if config_dict.get("disabled") is True:
+            return None
+
+        class_args = config_dict.get("args", dict())
+        class_args = self.process_args(class_args)
+
+        class_path = self.get_class_path(config_dict)
+
+        # Handle the special case, if config pass in the class_attributes, use the user defined class attributes
+        # parameters directly.
+        if "class_attributes" in class_args:
+            class_args = class_args["class_attributes"]
+
+        return instantiate_class(class_path, class_args)
+
+    def get_class_path(self, config_dict):
+        if "path" in config_dict.keys():
+            path_spec = config_dict["path"]
+            if not isinstance(path_spec, str):
+                raise ConfigError("path spec must be str")
+
+            if len(path_spec) <= 0:
+                raise ConfigError("path spec must not be empty")
+
+            class_path = format(path_spec)
+            parts = class_path.split(".")
+            if len(parts) < 2:
+                raise ConfigError("invalid class path '{}': missing module name".format(class_path))
+        else:
+            if "name" not in config_dict:
+                raise ConfigError("class name or path must be specified")
+
+            class_name = config_dict["name"]
+
+            if not isinstance(class_name, str):
+                raise ConfigError("class name must be str")
+
+            if len(class_name) <= 0:
+                raise ConfigError("class name must not be empty")
+            module_name = self.module_scanner.get_module_name(class_name)
+            if module_name is None:
+                raise ConfigError('Cannot find component class "{}"'.format(class_name))
+            class_path = module_name + ".{}".format(class_name)
+
+        return class_path
+
+    def is_configured_subclass(self, config_dict, base_class):
+        return issubclass(get_class(self.get_class_path(config_dict)), base_class)
+
+    def start_config(self, config_ctx: ConfigContext):
+        pass
+
+    def process_config_element(self, config_ctx: ConfigContext, node: Node):
+        pass
+
+    def finalize_config(self, config_ctx: ConfigContext):
+        pass
+
+
+def get_component_refs(component):
+    """Get component reference.
+
+    Args:
+        component: string for component
+
+    Returns: list of component and reference
+
+    """
+    if "name" in component:
+        name = component["name"]
+        key = "name"
+    elif "path" in component:
+        name = component["path"]
+        key = "path"
+    else:
+        raise ConfigError('component has no "name" or "path')
+
+    parts = name.split("#")
+    component[key] = parts[0]
+    return parts
diff --git a/nvflare/lighter/__init__.py b/nvflare/lighter/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/lighter/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/lighter/impl/__init__.py b/nvflare/lighter/impl/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/lighter/impl/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/lighter/impl/auth_policy.py b/nvflare/lighter/impl/auth_policy.py
new file mode 100644
index 0000000000..7e5e3cbb3e
--- /dev/null
+++ b/nvflare/lighter/impl/auth_policy.py
@@ -0,0 +1,57 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from nvflare.lighter.spec import Builder, Study
+
+
+class AuthPolicyBuilder(Builder):
+    def __init__(self, orgs, roles, groups, disabled):
+        self.orgs = orgs
+        self.roles = roles
+        self.groups = groups
+        self.disabled = disabled
+
+    def build(self, study: Study, ctx: dict):
+        authz = {"version": "1.0"}
+        authz["roles"] = self.roles
+        authz["groups"] = self.groups
+        users = dict()
+        for admin in study.get_participants_by_type("admin", first_only=False):
+            if admin.org not in self.orgs:
+                raise ValueError(f"Admin {admin.name}'s org {admin.org} not defined in AuthPolicy")
+            if self.disabled:
+                users[admin.name] = {"org": admin.org, "roles": ["super"]}
+            else:
+                for role in admin.props.get("roles"):
+                    if role not in self.roles:
+                        raise ValueError(f"Admin {admin.name}'s role {role} not defined in AuthPolicy")
+                users[admin.name] = {"org": admin.org, "roles": admin.props.get("roles")}
+        authz["users"] = users
+        authz["orgs"] = self.orgs
+        server = study.get_participants_by_type("server")
+        if server.org not in self.orgs:
+            raise ValueError(f"Server {server.name}'s org {server.org} not defined in AuthPolicy")
+        sites = {"server": server.org}
+        for client in study.get_participants_by_type("client", first_only=False):
+            if client.org not in self.orgs:
+                raise ValueError(f"client {client.name}'s org {client.org} not defined in AuthPolicy")
+            sites[client.name] = client.org
+        authz["sites"] = sites
+        authz.update(ctx.get("authz_def"))
+        dest_dir = self.get_kit_dir(server, ctx)
+        with open(os.path.join(dest_dir, "authorization.json"), "wt") as f:
+            f.write(json.dumps(authz))
diff --git a/nvflare/lighter/impl/cert.py b/nvflare/lighter/impl/cert.py
new file mode 100644
index 0000000000..bb8da0ba1a
--- /dev/null
+++ b/nvflare/lighter/impl/cert.py
@@ -0,0 +1,151 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import os
+import pickle
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
+
+from nvflare.lighter.spec import Builder
+
+
+def serialize_pri_key(pri_key):
+    return pri_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+
+
+def serialize_cert(cert):
+    return cert.public_bytes(serialization.Encoding.PEM)
+
+
+class CertBuilder(Builder):
+    def __init__(self):
+        self.root_cert = None
+        self.persistent_state = dict()
+
+    def initialize(self, ctx):
+        state_dir = self.get_state_dir(ctx)
+        cert_file = os.path.join(state_dir, "cert.pkl")
+        if os.path.exists(cert_file):
+            self.persistent_state = pickle.load(open(cert_file, "rb"))
+            self.serialized_cert = self.persistent_state["root_cert"]
+            self.root_cert = x509.load_pem_x509_certificate(self.serialized_cert, default_backend())
+            self.pri_key = serialization.load_pem_private_key(
+                self.persistent_state["root_pri_key"], password=None, backend=default_backend()
+            )
+            self.pub_key = self.pri_key.public_key()
+            self.subject = self.root_cert.subject
+
+    def _build_root(self, subject):
+        if not self.persistent_state:
+            pri_key, pub_key = self._generate_keys()
+            self.subject = self._x509_name(subject)
+            issuer = self.subject
+            self.root_cert = self._generate_cert(self.subject, issuer, pri_key, pub_key, ca=True)
+            self.pri_key = pri_key
+            self.pub_key = pub_key
+            self.serialized_cert = serialize_cert(self.root_cert)
+            self.persistent_state["root_cert"] = self.serialized_cert
+            self.persistent_state["root_pri_key"] = serialize_pri_key(self.pri_key)
+
+    def _build_write_cert_pair(self, participant, base_name, ctx):
+        subject = participant.subject
+        if self.persistent_state and subject in self.persistent_state:
+            cert = x509.load_pem_x509_certificate(self.persistent_state[subject]["cert"], default_backend())
+            pri_key = serialization.load_pem_private_key(
+                self.persistent_state[subject]["pri_key"], password=None, backend=default_backend()
+            )
+        else:
+            pri_key, cert = self.get_pri_key_cert(participant)
+            self.persistent_state[subject] = dict(cert=serialize_cert(cert), pri_key=serialize_pri_key(pri_key))
+        dest_dir = self.get_kit_dir(participant, ctx)
+        with open(os.path.join(dest_dir, f"{base_name}.crt"), "wb") as f:
+            f.write(serialize_cert(cert))
+        with open(os.path.join(dest_dir, f"{base_name}.key"), "wb") as f:
+            f.write(serialize_pri_key(pri_key))
+        with open(os.path.join(dest_dir, "rootCA.pem"), "wb") as f:
+            f.write(self.serialized_cert)
+
+    def build(self, study, ctx):
+        self._build_root(study.name)
+        ctx["root_cert"] = self.root_cert
+        ctx["root_pri_key"] = self.pri_key
+        server = study.get_participants_by_type("server")
+        self._build_write_cert_pair(server, "server", ctx)
+
+        for client in study.get_participants_by_type("client", first_only=False):
+            self._build_write_cert_pair(client, "client", ctx)
+
+        for admin in study.get_participants_by_type("admin", first_only=False):
+            self._build_write_cert_pair(admin, "client", ctx)
+
+    def get_pri_key_cert(self, participant):
+        pri_key, pub_key = self._generate_keys()
+        subject = self._x509_name(participant.subject)
+        issuer = self.subject
+        cert = self._generate_cert(subject, issuer, self.pri_key, pub_key)
+        return pri_key, cert
+
+    def _generate_keys(self):
+        pri_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend())
+        pub_key = pri_key.public_key()
+        return pri_key, pub_key
+
+    def _generate_cert(self, subject, issuer, signing_pri_key, subject_pub_key, valid_days=360, ca=False):
+        builder = (
+            x509.CertificateBuilder()
+            .subject_name(subject)
+            .issuer_name(issuer)
+            .public_key(subject_pub_key)
+            .serial_number(x509.random_serial_number())
+            .not_valid_before(datetime.datetime.utcnow())
+            .not_valid_after(
+                # Our certificate will be valid for 360 days
+                datetime.datetime.utcnow()
+                + datetime.timedelta(days=valid_days)
+                # Sign our certificate with our private key
+            )
+        )
+        if ca:
+            builder = (
+                builder.add_extension(
+                    x509.SubjectKeyIdentifier.from_public_key(subject_pub_key),
+                    critical=False,
+                )
+                .add_extension(
+                    x509.AuthorityKeyIdentifier.from_issuer_public_key(subject_pub_key),
+                    critical=False,
+                )
+                .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=False)
+            )
+        return builder.sign(signing_pri_key, hashes.SHA256(), default_backend())
+
+    def _x509_name(self, cn_name, org_name=None):
+        name = [x509.NameAttribute(NameOID.COMMON_NAME, cn_name)]
+        if org_name is not None:
+            name.append(x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name))
+        return x509.Name(name)
+
+    def finalize(self, ctx):
+        state_dir = self.get_state_dir(ctx)
+        cert_file = os.path.join(state_dir, "cert.pkl")
+        pickle.dump(self.persistent_state, open(cert_file, "wb"))
diff --git a/nvflare/lighter/impl/he.py b/nvflare/lighter/impl/he.py
new file mode 100644
index 0000000000..7e54e3a5b9
--- /dev/null
+++ b/nvflare/lighter/impl/he.py
@@ -0,0 +1,73 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import tenseal as ts
+
+from nvflare.lighter.spec import Builder
+
+
+class HEBuilder(Builder):
+    def __init__(
+        self,
+        poly_modulus_degree=8192,
+        coeff_mod_bit_sizes=[60, 40, 40],
+        scale_bits=40,
+        scheme="CKKS",
+    ):
+
+        self._context = None
+        self.scheme_type_mapping = {
+            "CKKS": ts.SCHEME_TYPE.CKKS,
+            "BFV": ts.SCHEME_TYPE.BFV,
+        }
+        self.poly_modulus_degree = poly_modulus_degree
+        self.coeff_mod_bit_sizes = coeff_mod_bit_sizes
+        self.scale_bits = scale_bits
+        _scheme = scheme
+        # Setup TenSEAL context
+        self.scheme_type = self.scheme_type_mapping[_scheme]
+        self.serialized = None
+
+    def initialize(self, ctx):
+        self._context = ts.context(
+            self.scheme_type,
+            poly_modulus_degree=self.poly_modulus_degree,
+            coeff_mod_bit_sizes=self.coeff_mod_bit_sizes,
+            encryption_type=ts.ENCRYPTION_TYPE.SYMMETRIC,
+        )
+        # dynamically call different generate keys method
+        # getattr(self._context, f'generate_{self.key_type}_keys')()
+        self._context.generate_relin_keys()
+        self._context.global_scale = 2 ** self.scale_bits
+
+    def build(self, study, ctx):
+        server = study.get_participants_by_type("server")
+        dest_dir = self.get_kit_dir(server, ctx)
+        with open(os.path.join(dest_dir, "server_context.tenseal"), "wb") as f:
+            f.write(self.get_serialized_context())
+        for client in study.get_participants_by_type("client", first_only=False):
+            dest_dir = self.get_kit_dir(client, ctx)
+            with open(os.path.join(dest_dir, "client_context.tenseal"), "wb") as f:
+                f.write(self.get_serialized_context(is_client=True))
+
+    def get_serialized_context(self, is_client=False):
+        _serialized_context = self._context.serialize(
+            save_public_key=is_client,
+            save_secret_key=is_client,
+            save_galois_keys=False,
+            save_relin_keys=True,
+        )
+        return _serialized_context
diff --git a/nvflare/lighter/impl/master_template.yml b/nvflare/lighter/impl/master_template.yml
new file mode 100644
index 0000000000..55dd86edfe
--- /dev/null
+++ b/nvflare/lighter/impl/master_template.yml
@@ -0,0 +1,466 @@
+readme_am: |
+  *********************************
+  Admin Client package
+  *********************************
+  The package includes at least the following files:
+  readme.txt
+  rootCA.pem
+  client.crt
+  client.key
+  fl_admin.sh
+  
+  Please install the nvflare package by 'python3 -m pip nvflare.'  This will install a set of Python codes
+  in your environment.  After installation, you can run the fl_admin.sh file to start communicating to the admin server.
+
+  The rootCA.pem file is pointed by "ca_cert" in fl_admin.sh.  If you plan to move/copy it to a different place,
+  you will need to modify fl_admin.sh.  The same applies to the other two files, client.crt and client.key.
+
+  The email in your submission to participate this Federated Learning project is embedded in the CN field of client
+  certificate, which uniquely identifies the participant.  As such, please safeguard its private key, client.key.
+
+readme_fc: |
+  *********************************
+  Federated Learning Client package
+  *********************************
+  The package includes at least the following files:
+  readme.txt
+  rootCA.pem
+  client.crt
+  client.key
+  fed_client.json
+  start.sh
+  sub_start.sh
+  stop_fl.sh
+
+  Run start.sh to start the client.
+
+  The rootCA.pem file is pointed by "ssl_root_cert" in fed_client.json.  If you plan to move/copy it to a different place,
+  you will need to modify fed_client.json.  The same applies to the other two files, client.crt and client.key.
+
+  The client name in your submission to participate this Federated Learning project is embedded in the CN field of client
+  certificate, which uniquely identifies the participant.  As such, please safeguard its private key, client.key.
+
+readme_fs: |
+  *********************************
+  Federated Learning Server package
+  *********************************
+  The package includes at least the following files:
+  readme.txt
+  rootCA.pem
+  server.crt
+  server.key
+  authorization.json
+  fed_server.json
+  start.sh
+  sub_start.sh
+  stop_fl.sh
+  signature.pkl
+
+  Run start.sh to start the server.
+
+  The rootCA.pem file is pointed by "ssl_root_cert" in fed_server.json.  If you plan to move/copy it to a different place,
+  you will need to modify fed_server.json.  The same applies to the other two files, server.crt and server.key.
+
+  Please always safeguard the server.key.
+
+fed_client: |
+  {
+    "format_version": 1,
+    "enable_byoc": false,
+    "servers": [
+        {
+            "name": "spleen_segmentation",
+            "service": {
+                "target": "localhost:8002",
+                "options": [
+                    ["grpc.max_send_message_length",    2147483647],
+                    ["grpc.max_receive_message_length", 2147483647]
+                ]
+            }
+        }
+    ],
+    "client": {
+        "retry_timeout": 30,
+        "ssl_private_key": "client.key",
+        "ssl_cert": "client.crt",
+        "ssl_root_cert": "rootCA.pem",
+        "compression": "Gzip"
+    }
+  }
+
+fed_server: |
+  {
+    "format_version": 1,
+    "enable_byoc": false,
+    "servers": [
+        {
+            "name": "spleen_segmentation",
+            "service": {
+                "target": "localhost:8002",
+                "options": [
+                    ["grpc.max_send_message_length",    2147483647],
+                    ["grpc.max_receive_message_length", 2147483647]
+                ]
+            },
+            "admin_host": "localhost",
+            "admin_port": 5005,
+            "admin_storage": "transfer",
+            "ssl_private_key": "server.key",
+            "ssl_cert": "server.crt",
+            "ssl_root_cert": "rootCA.pem",
+            "max_num_clients": 100,
+            "heart_beat_timeout": 600,
+            "num_server_workers": 4,
+            "compression": "Gzip"
+        }
+    ] 
+  }
+
+authz_def: |
+  {
+    "rules": {
+      "allow_byoc": {
+        "desc": "allow BYOC in APP",
+        "type": "bool",
+        "default": false
+      },
+      "allow_custom_datalist": {
+        "desc": "allow custom datalist",
+        "default": true
+      }
+    },
+    "rights": {
+      "upload_app": {
+        "desc": "upload app",
+        "default": true
+      },
+      "deploy_self": {
+        "desc": "deploy app to own site",
+        "default": false,
+        "precond": "selfOrg"
+      },
+      "deploy_all": {
+        "desc": "deploy app to all sites",
+        "default": false
+      },
+      "train_self": {
+        "desc": "do training operations on own site",
+        "default": false,
+        "precond": "selfOrg"
+      },
+      "train_all": {
+        "desc": "do training ops on all sites",
+        "default": false
+      },
+      "view_self": {
+        "desc": "view log files of own org",
+        "default": true,
+        "precond": "selfOrg"
+      },
+      "view_all": {
+        "desc": "view log files of all sites",
+        "default": false
+      },
+      "operate_all": {
+        "desc": "start/stop all sites",
+        "default": false
+      },
+      "operate_self": {
+        "desc": "start/stop own site",
+        "default": false,
+        "precond": "selfOrg"
+      }
+    }
+  }
+
+fl_admin_sh: |
+  #!/usr/bin/env bash
+  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  mkdir -p $DIR/../transfer
+  python3 -m nvflare.fuel.hci.tools.admin --host {~~cn~~} --port {~~admin_port~~} --prompt "> " --with_file_transfer --upload_dir=$DIR/../transfer --download_dir=$DIR/../transfer --with_shell --with_login --with_ssl --cred_type cert --ca_cert=$DIR/rootCA.pem --client_cert=$DIR/client.crt --client_key=$DIR/client.key
+
+log_config: |
+  [loggers]
+  keys=root,modelLogger
+
+  [handlers]
+  keys=consoleHandler
+
+  [formatters]
+  keys=fullFormatter
+
+  [logger_root]
+  level=INFO
+  handlers=consoleHandler
+
+  [logger_modelLogger]
+  level=DEBUG
+  handlers=consoleHandler
+  qualname=modelLogger
+  propagate=0
+
+  [handler_consoleHandler]
+  class=StreamHandler
+  level=DEBUG
+  formatter=fullFormatter
+  args=(sys.stdout,)
+
+  [formatter_fullFormatter]
+  format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+
+start_cln_sh: |
+  #!/usr/bin/env bash
+  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  $DIR/sub_start.sh &
+
+start_svr_sh: |
+  #!/usr/bin/env bash
+  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  $DIR/sub_start.sh &
+
+stop_fl_sh: |
+  #!/usr/bin/env bash
+  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  echo "Shutdown request created.  Wait for local FL process to shutdown."
+  touch $DIR/../shutdown.fl
+  sleep 5
+  if [[ ! -f "$DIR/../pid.fl" ]]; then
+    echo "No pid.fl.  No need to kill process."
+    exit
+  fi
+  pid=`cat $DIR/../pid.fl`
+  kill -0 ${pid} 2> /dev/null 1>&2
+  if [[ $? -ne 0 ]]; then
+    echo "Process already terminated"
+    exit
+  fi
+  kill -9 $pid
+  pid=`cat $DIR/../tee.fl`
+  kill -9 $pid
+  rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../tee.fl
+  echo "Shutdown process finished."
+
+sub_start_cln_sh: |
+  #!/usr/bin/env bash
+  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  echo "WORKSPACE set to $DIR/.."
+  mkdir -p $DIR/../transfer
+  export PYTHONPATH=/local/custom:$PYTHONPATH
+  echo "PYTHONPATH is $PYTHONPATH"
+
+  SECONDS=0
+  lst=-400
+  restart_count=0
+  start_fl() {
+    if [[ $(( $SECONDS - $lst )) -lt 300 ]]; then
+      ((restart_count++))
+    else
+      restart_count=0
+    fi
+    if [[ $(($SECONDS - $lst )) -lt 300 && $restart_count -ge 5 ]]; then
+      echo "System is in trouble and unable to start the task!!!!!"
+      rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl
+      exit
+    fi
+    lst=$SECONDS
+  ((python3 -u -m nvflare.private.fed.app.client.client_train -m $DIR/.. -s fed_client.json --set secure_train=true multi_gpu=false uid={~~client_name~~} config_folder={~~config_folder~~} 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | (tee -a $DIR/../log.txt & echo $! >&4 ) 4>$DIR/../tee.fl )
+    pid=`cat $DIR/../pid.fl`
+    echo "new pid ${pid}"
+  }
+
+  stop_fl() {
+    if [[ ! -f "$DIR/../pid.fl" ]]; then
+      echo "No pid.fl.  No need to kill process."
+      return
+    fi
+    pid=`cat $DIR/../pid.fl`
+    sleep 5
+    kill -0 ${pid} 2> /dev/null 1>&2
+    if [[ $? -ne 0 ]]; then
+      echo "Process already terminated"
+      return
+    fi
+    kill -9 $pid
+    pid=`cat $DIR/../tee.fl`
+    kill -9 $pid
+    rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../tee.fl
+  }
+    
+  if [[ -f "$DIR/../daemon_pid.fl" ]]; then
+    dpid=`cat $DIR/../daemon_pid.fl`
+    kill -0 ${dpid} 2> /dev/null 1>&2
+    if [[ $? -eq 0 ]]; then
+      echo "There seems to be one instance, pid=$dpid, running."
+      echo "If you are sure it's not the case, please kill process $dpid and then remove daemon_pid.fl in $DIR/.."
+      exit
+    fi
+    rm -f $DIR/../daemon_pid.fl
+  fi
+
+  echo $BASHPID > $DIR/../daemon_pid.fl
+
+  while true
+  do
+    sleep 5
+    if [[ ! -f "$DIR/../pid.fl" ]]; then
+      echo "start fl because of no pid.fl"
+      start_fl
+      continue
+    fi
+    pid=`cat $DIR/../pid.fl`
+    kill -0 ${pid} 2> /dev/null 1>&2
+    if [[ $? -ne 0 ]]; then
+      if [[ -f "$DIR/../shutdown.fl" ]]; then
+        echo "Gracefully shutdown."
+        break
+      fi
+      echo "start fl because process of ${pid} does not exist"
+      start_fl
+      continue
+    fi
+    if [[ -f "$DIR/../shutdown.fl" ]]; then
+      echo "About to shutdown."
+      stop_fl
+      break
+    fi
+    if [[ -f "$DIR/../restart.fl" ]]; then
+      echo "About to restart."
+      stop_fl
+    fi
+  done
+
+  rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl $DIR/../tee.fl
+
+sub_start_svr_sh: |
+  #!/usr/bin/env bash
+  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  export CUDA_VISIBLE_DEVICES=
+  echo "WORKSPACE set to $DIR/.."
+  mkdir -p $DIR/../transfer 
+
+  SECONDS=0
+  lst=-400
+  restart_count=0
+  start_fl() {
+    if [[ $(( $SECONDS - $lst )) -lt 300 ]]; then
+      ((restart_count++))
+    else
+      restart_count=0
+    fi
+    if [[ $(($SECONDS - $lst )) -lt 300 && $restart_count -ge 5 ]]; then
+      echo "System is in trouble and unable to start the task!!!!!"
+      rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl
+      exit
+    fi
+    lst=$SECONDS
+  ((python3 -u -m nvflare.private.fed.app.server.server_train -m $DIR/.. -s fed_server.json --set secure_train=true config_folder={~~config_folder~~} 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | (tee -a $DIR/../log.txt & echo $! >&4 ) 4>$DIR/../tee.fl )
+    pid=`cat $DIR/../pid.fl`
+    echo "new pid ${pid}"
+  }
+
+  stop_fl() {
+    if [[ ! -f "$DIR/../pid.fl" ]]; then
+      echo "No pid.fl.  No need to kill process."
+      return
+    fi
+    pid=`cat $DIR/../pid.fl`
+    sleep 5
+    kill -0 ${pid} 2> /dev/null 1>&2
+    if [[ $? -ne 0 ]]; then
+      echo "Process already terminated"
+      return
+    fi
+    kill -9 $pid
+    pid=`cat $DIR/../tee.fl`
+    kill -9 $pid
+    rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../tee.fl
+  }
+    
+  if [[ -f "$DIR/../daemon_pid.fl" ]]; then
+    dpid=`cat $DIR/../daemon_pid.fl`
+    kill -0 ${dpid} 2> /dev/null 1>&2
+    if [[ $? -eq 0 ]]; then
+      echo "There seems to be one instance, pid=$dpid, running."
+      echo "If you are sure it's not the case, please kill process $dpid and then remove daemon_pid.fl in $DIR/.."
+      exit
+    fi
+    rm -f $DIR/../daemon_pid.fl
+  fi
+
+  echo $BASHPID > $DIR/../daemon_pid.fl
+
+  while true
+  do
+    sleep 5
+    if [[ ! -f "$DIR/../pid.fl" ]]; then
+      echo "start fl because of no pid.fl"
+      start_fl
+      continue
+    fi
+    pid=`cat $DIR/../pid.fl`
+    kill -0 ${pid} 2> /dev/null 1>&2
+    if [[ $? -ne 0 ]]; then
+      if [[ -f "$DIR/../shutdown.fl" ]]; then
+        echo "Gracefully shutdown."
+        break
+      fi
+      echo "start fl because process of ${pid} does not exist"
+      start_fl
+      continue
+    fi
+    if [[ -f "$DIR/../shutdown.fl" ]]; then
+      echo "About to shutdown."
+      stop_fl
+      break
+    fi
+    if [[ -f "$DIR/../restart.fl" ]]; then
+      echo "About to restart."
+      stop_fl
+    fi
+  done
+
+  rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl $DIR/../tee.fl
+
+docker_cln_sh: |
+  #!/usr/bin/env bash
+  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  # docker run script for FL client
+  # local data directory
+  MY_DATA_DIR=/home/flclient/data
+  # for all gpus use line below 
+  GPU2USE=all 
+  # for 2 gpus use line below
+  #GPU2USE=2 
+  # for specific gpus as gpu#0 and gpu#2 use line below
+  #GPU2USE='"device=0,2"'
+  # to use host network, use line below
+  NETARG="--net=host"
+  # FL clients do not need to open ports, so the following line is not needed.
+  #NETARG="-p 443:443 -p 8003:8003"
+  DOCKER_IMAGE={~~docker_image~~}
+  echo "Starting docker with $DOCKER_IMAGE"
+  docker run --rm -it --name={~~client_name~~} --gpus=$GPU2USE -u $(id -u):$(id -g) -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group -v $DIR/..:/workspace/ -v $MY_DATA_DIR:/data/:ro -w /workspace/ --ipc=host $NETARG $DOCKER_IMAGE /bin/bash
+
+docker_svr_sh: |
+  #!/usr/bin/env bash
+  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  # docker run script for FL server
+  # to use host network, use line below
+  NETARG="--net=host"
+  # or to expose specific ports, use line below
+  #NETARG="-p {~~admin_port~~}:{~~admin_port~~} -p {~~fed_learn_port~~}:{~~fed_learn_port~~}"
+  DOCKER_IMAGE={~~docker_image~~}
+  echo "Starting docker with $DOCKER_IMAGE"
+  docker run --rm -it --name=flserver -v $DIR/..:/workspace/ -w /workspace/ --ipc=host $NETARG $DOCKER_IMAGE /bin/bash
+
+docker_adm_sh: |
+  #!/usr/bin/env bash
+  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  # docker run script for FL admin
+  # to use host network, use line below
+  #NETARG="--net=host"
+  # Admin clients do not need to open ports, so the following line is not needed.
+  #NETARG="-p 8003:8003"
+  DOCKER_IMAGE={~~docker_image~~}
+  echo "Starting docker with $DOCKER_IMAGE"
+  docker run --rm -it --name=fladmin -v $DIR/..:/workspace/ -w /workspace/ $DOCKER_IMAGE /bin/bash
diff --git a/nvflare/lighter/impl/signature.py b/nvflare/lighter/impl/signature.py
new file mode 100644
index 0000000000..eff9dd8499
--- /dev/null
+++ b/nvflare/lighter/impl/signature.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pickle
+
+from nvflare.lighter.spec import Builder, Study
+from nvflare.lighter.utils import sign_all
+
+
+class SignatureBuilder(Builder):
+    def build(self, study: Study, ctx: dict):
+        server = study.get_participants_by_type("server")
+        dest_dir = self.get_kit_dir(server, ctx)
+        root_pri_key = ctx.get("root_pri_key")
+        signatures = sign_all(dest_dir, root_pri_key)
+        pickle.dump(signatures, open(os.path.join(dest_dir, "signature.pkl"), "wb"))
+        for p in study.get_participants_by_type("client", first_only=False):
+            dest_dir = self.get_kit_dir(p, ctx)
+            root_pri_key = ctx.get("root_pri_key")
+            signatures = sign_all(dest_dir, root_pri_key)
+            pickle.dump(signatures, open(os.path.join(dest_dir, "signature.pkl"), "wb"))
diff --git a/nvflare/lighter/impl/static_file.py b/nvflare/lighter/impl/static_file.py
new file mode 100644
index 0000000000..ae228dc473
--- /dev/null
+++ b/nvflare/lighter/impl/static_file.py
@@ -0,0 +1,182 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from nvflare.lighter.spec import Builder
+from nvflare.lighter.utils import sh_replace
+
+
+class StaticFileBuilder(Builder):
+    def __init__(self, enable_byoc=False, config_folder="", docker_image=""):
+        self.enable_byoc = enable_byoc
+        self.config_folder = config_folder
+        self.docker_image = docker_image
+
+    def _write(self, file_full_path, content, mode, exe=False):
+        mode = mode + "w"
+        with open(file_full_path, mode) as f:
+            f.write(content)
+        if exe:
+            os.chmod(file_full_path, 0o755)
+
+    def _build_server(self, server, ctx):
+        config = json.loads(self.template["fed_server"])
+        dest_dir = self.get_kit_dir(server, ctx)
+        server_0 = config["servers"][0]
+        server_0["name"] = self.study_name
+        admin_port = server.props.get("admin_port", 8003)
+        ctx["admin_port"] = admin_port
+        fed_learn_port = server.props.get("fed_learn_port", 8002)
+        ctx["fed_learn_port"] = fed_learn_port
+        ctx["server_name"] = server.name
+        server_0["service"]["target"] = f"{server.name}:{fed_learn_port}"
+        server_0["admin_host"] = server.name
+        server_0["admin_port"] = admin_port
+        config["enable_byoc"] = server.enable_byoc
+        self._write(os.path.join(dest_dir, "fed_server.json"), json.dumps(config), "t")
+        replacement_dict = {
+            "admin_port": admin_port,
+            "fed_learn_port": fed_learn_port,
+            "config_folder": self.config_folder,
+            "docker_image": self.docker_image,
+        }
+        if self.docker_image:
+            self._write(
+                os.path.join(dest_dir, "docker.sh"),
+                sh_replace(self.template["docker_svr_sh"], replacement_dict),
+                "t",
+                exe=True,
+            )
+        self._write(
+            os.path.join(dest_dir, "start.sh"),
+            self.template["start_svr_sh"],
+            "t",
+            exe=True,
+        )
+        self._write(
+            os.path.join(dest_dir, "sub_start.sh"),
+            sh_replace(self.template["sub_start_svr_sh"], replacement_dict),
+            "t",
+            exe=True,
+        )
+        self._write(
+            os.path.join(dest_dir, "log.config"),
+            self.template["log_config"],
+            "t",
+        )
+        self._write(
+            os.path.join(dest_dir, "readme.txt"),
+            self.template["readme_fs"],
+            "t",
+        )
+        self._write(
+            os.path.join(dest_dir, "stop_fl.sh"),
+            self.template["stop_fl_sh"],
+            "t",
+            exe=True,
+        )
+
+    def _build_client(self, client, ctx):
+        config = json.loads(self.template["fed_client"])
+        dest_dir = self.get_kit_dir(client, ctx)
+        fed_learn_port = ctx.get("fed_learn_port")
+        server_name = ctx.get("server_name")
+        config["servers"][0]["service"]["target"] = f"{server_name}:{fed_learn_port}"
+        config["servers"][0]["name"] = self.study_name
+        config["enable_byoc"] = client.enable_byoc
+        replacement_dict = {
+            "client_name": f"{client.subject}",
+            "config_folder": self.config_folder,
+            "docker_image": self.docker_image,
+        }
+
+        self._write(os.path.join(dest_dir, "fed_client.json"), json.dumps(config), "t")
+        if self.docker_image:
+            self._write(
+                os.path.join(dest_dir, "docker.sh"),
+                sh_replace(self.template["docker_cln_sh"], replacement_dict),
+                "t",
+                exe=True,
+            )
+        self._write(
+            os.path.join(dest_dir, "start.sh"),
+            self.template["start_cln_sh"],
+            "t",
+            exe=True,
+        )
+        self._write(
+            os.path.join(dest_dir, "sub_start.sh"),
+            sh_replace(self.template["sub_start_cln_sh"], replacement_dict),
+            "t",
+            exe=True,
+        )
+        self._write(
+            os.path.join(dest_dir, "log.config"),
+            self.template["log_config"],
+            "t",
+        )
+        self._write(
+            os.path.join(dest_dir, "readme.txt"),
+            self.template["readme_fc"],
+            "t",
+        )
+        self._write(
+            os.path.join(dest_dir, "stop_fl.sh"),
+            self.template["stop_fl_sh"],
+            "t",
+            exe=True,
+        )
+
+    def _build_admin(self, admin, ctx):
+        dest_dir = self.get_kit_dir(admin, ctx)
+        admin_port = ctx.get("admin_port")
+        server_name = ctx.get("server_name")
+
+        replacement_dict = {
+            "cn": f"{server_name}",
+            "admin_port": f"{admin_port}",
+            "docker_image": self.docker_image,
+        }
+        if self.docker_image:
+            self._write(
+                os.path.join(dest_dir, "docker.sh"),
+                sh_replace(self.template["docker_adm_sh"], replacement_dict),
+                "t",
+                exe=True,
+            )
+        self._write(
+            os.path.join(dest_dir, "fl_admin.sh"),
+            sh_replace(self.template["fl_admin_sh"], replacement_dict),
+            "t",
+            exe=True,
+        )
+        self._write(
+            os.path.join(dest_dir, "readme.txt"),
+            self.template["readme_am"],
+            "t",
+        )
+
+    def build(self, study, ctx):
+        self.template = ctx.get("template")
+        server = study.get_participants_by_type("server")
+        self.study_name = study.name
+        self._build_server(server, ctx)
+
+        for client in study.get_participants_by_type("client", first_only=False):
+            self._build_client(client, ctx)
+
+        for admin in study.get_participants_by_type("admin", first_only=False):
+            self._build_admin(admin, ctx)
diff --git a/nvflare/lighter/impl/template.py b/nvflare/lighter/impl/template.py
new file mode 100644
index 0000000000..9438b1093b
--- /dev/null
+++ b/nvflare/lighter/impl/template.py
@@ -0,0 +1,29 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from nvflare.lighter.spec import Builder
+from nvflare.lighter.utils import load_yaml
+
+
+class TemplateBuilder(Builder):
+    def initialize(self, ctx):
+        resource_dir = self.get_resources_dir(ctx)
+        template_file = ctx.get("template_file")
+        template = load_yaml(os.path.join(resource_dir, template_file))
+        authz_def = json.loads(template.get("authz_def"))
+        ctx["template"] = template
+        ctx["authz_def"] = authz_def
diff --git a/nvflare/lighter/impl/workspace.py b/nvflare/lighter/impl/workspace.py
new file mode 100644
index 0000000000..5acaeec752
--- /dev/null
+++ b/nvflare/lighter/impl/workspace.py
@@ -0,0 +1,81 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pathlib
+import shutil
+import subprocess
+
+from nvflare.lighter.spec import Builder, Study
+from nvflare.lighter.utils import generate_password
+
+
+class WorkspaceBuilder(Builder):
+    def __init__(self, template_file):
+        self.template_file = template_file
+
+    def _make_dir(self, dirs):
+        for dir in dirs:
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+
+    def initialize(self, ctx):
+        workspace_dir = ctx["workspace"]
+        prod_dirs = [_ for _ in os.listdir(workspace_dir) if _.startswith("prod_")]
+        last = -1
+        for dir in prod_dirs:
+            stage = int(dir.split("_")[-1])
+            if stage > last:
+                last = stage
+        ctx["last_prod_stage"] = last
+        template_file_full_path = os.path.join(self.get_resources_dir(ctx), self.template_file)
+        file_path = pathlib.Path(__file__).parent.absolute()
+        shutil.copyfile(os.path.join(file_path, self.template_file), template_file_full_path)
+        ctx["template_file"] = self.template_file
+
+    def build(self, study: Study, ctx: dict):
+        dirs = [self.get_kit_dir(p, ctx) for p in study.participants]
+        self._make_dir(dirs)
+
+    def finalize(self, ctx: dict):
+        if ctx["last_prod_stage"] >= 99:
+            print(f"Please clean up {ctx['workspace']} by removing prod_N folders")
+            print("After clean-up, rerun the provision command.")
+        else:
+            current_prod_stage = str(ctx["last_prod_stage"] + 1).zfill(2)
+            current_prod_dir = os.path.join(ctx["workspace"], f"prod_{current_prod_stage}")
+            shutil.move(self.get_wip_dir(ctx), current_prod_dir)
+            ctx.pop("wip_dir", None)
+            print(f"Generated results can be found under {current_prod_dir}.  Builder's wip folder removed.")
+            ctx["current_prod_dir"] = current_prod_dir
+
+
+class DistributionBuilder(Builder):
+    def __init__(self, zip_password=False):
+        self.zip_password = zip_password
+
+    def build(self, study: Study, ctx: dict):
+        wip_dir = self.get_wip_dir(ctx)
+        dirs = [name for name in os.listdir(wip_dir) if os.path.isdir(os.path.join(wip_dir, name))]
+        for dir in dirs:
+            dest_zip_file = os.path.join(wip_dir, f"{dir}")
+            if self.zip_password:
+                pw = generate_password()
+                run_args = ["zip", "-rq", "-P", pw, dest_zip_file + ".zip", ".", "-i", "startup/*"]
+                os.chdir(dest_zip_file)
+                subprocess.run(run_args)
+                os.chdir(os.path.join(dest_zip_file, ".."))
+                print(f"Password {pw} on {dir}.zip")
+            else:
+                shutil.make_archive(dest_zip_file, "zip", root_dir=os.path.join(wip_dir, dir), base_dir="startup")
diff --git a/nvflare/lighter/poc.py b/nvflare/lighter/poc.py
new file mode 100644
index 0000000000..29a8f8d126
--- /dev/null
+++ b/nvflare/lighter/poc.py
@@ -0,0 +1,59 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import pathlib
+import shutil
+import subprocess
+
+
+def clone_client(num_clients: int):
+    current_path = os.getcwd()
+    poc_folder = os.path.join(current_path, "poc")
+    src_folder = os.path.join(poc_folder, "client")
+    for index in range(1, num_clients + 1):
+        dst_folder = os.path.join(poc_folder, f"site-{index}")
+        shutil.copytree(src_folder, dst_folder)
+        start_sh = open(os.path.join(dst_folder, "startup", "start.sh"), "rt")
+        content = start_sh.read()
+        start_sh.close()
+        content = content.replace("NNN", f"{index}")
+        with open(os.path.join(dst_folder, "startup", "start.sh"), "wt") as f:
+            f.write(content)
+    shutil.rmtree(src_folder)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-n", "--num_clients", type=int, default=1, help="number of client folders to create")
+
+    args = parser.parse_args()
+
+    file_path = pathlib.Path(__file__).parent.absolute()
+    poc_zip_path = os.path.join(file_path, "..", "poc.zip")
+    answer = input("This will delete poc folder in current directory and create a new one. Is it OK to proceed? (y/N) ")
+    if answer.strip().upper() == "Y":
+        shutil.rmtree(os.path.join(os.getcwd(), "poc"), ignore_errors=True)
+        completed_process = subprocess.run(["unzip", "-q", poc_zip_path])
+        returncode = completed_process.returncode
+        if returncode != 0:
+            print(f"Error during creating poc folder: {returncode=}")
+            exit(returncode)
+        clone_client(args.num_clients)
+        print("Successfully creating poc folder.  Please read poc/Readme.rst for user guide.")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/nvflare/lighter/project.yml b/nvflare/lighter/project.yml
new file mode 100644
index 0000000000..589637638c
--- /dev/null
+++ b/nvflare/lighter/project.yml
@@ -0,0 +1,64 @@
+api_version: 2
+name: example_project
+description: NVFlare sample project yaml file
+
+participants:
+  # change example.com to the FQDN of the server
+  - name: example.com
+    type: server
+    org: nvidia
+    fed_learn_port: 8002
+    admin_port: 8003
+    # enable_byoc loads python codes in app.  Default is false.
+    enable_byoc: true
+  - name: site1
+    type: client
+    org: nvidia
+    enable_byoc: true
+  - name: site2
+    type: client
+    org: nvidia
+    enable_byoc: false
+  - name: admin@nvidia.com
+    type: admin
+    org: nvidia
+    roles:
+      - super
+
+# The same methods in all builders are called in their order defined in builders section
+builders:
+  - path: nvflare.lighter.impl.workspace.WorkspaceBuilder
+    args:
+      template_file: master_template.yml
+  - path: nvflare.lighter.impl.template.TemplateBuilder
+  - path: nvflare.lighter.impl.static_file.StaticFileBuilder
+    args:
+      # config_folder can be set to inform NVFlare where to get configuration
+      config_folder: config
+      # when docker_image is set to a docker image name, docker.sh will be generated on server/client/admin
+      # docker_image:
+  - path: nvflare.lighter.impl.auth_policy.AuthPolicyBuilder
+    args:
+      orgs:
+        nvidia:
+          - relaxed
+      roles:
+        super: super user of system
+      groups: 
+        relaxed: 
+          desc: org group with relaxed policies
+          rules: 
+            allow_byoc: true
+            allow_custom_datalist: true
+      disabled: false
+  - path: nvflare.lighter.impl.cert.CertBuilder
+  - path: nvflare.lighter.impl.he.HEBuilder
+    args:
+      poly_modulus_degree: 8192 
+      coeff_mod_bit_sizes: [60, 40, 40]
+      scale_bits: 40
+      scheme: CKKS
+  - path: nvflare.lighter.impl.signature.SignatureBuilder
+  - path: nvflare.lighter.impl.workspace.DistributionBuilder
+    args:
+      zip_password: false
diff --git a/nvflare/lighter/provision.py b/nvflare/lighter/provision.py
new file mode 100644
index 0000000000..2e7c34748c
--- /dev/null
+++ b/nvflare/lighter/provision.py
@@ -0,0 +1,85 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import argparse
+import os
+import pathlib
+import shutil
+import sys
+
+import yaml
+
+from nvflare.fuel.utils.class_utils import instantiate_class
+from nvflare.lighter.spec import Participant, Provisioner, Study
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-p", "--project_file", type=str, default="project.yml", help="file to describe FL project")
+    parser.add_argument("-w", "--workspace", type=str, default="workspace", help="directory used by provision")
+    parser.add_argument("-c", "--custom_folder", type=str, default=".", help="additional folder to load python codes")
+
+    args = parser.parse_args()
+
+    file_path = pathlib.Path(__file__).parent.absolute()
+    current_path = os.getcwd()
+    custom_folder_path = os.path.join(current_path, args.custom_folder)
+    sys.path.append(custom_folder_path)
+    print(f"Path list for python codes loading: {sys.path=}")
+
+    # main project file
+    project_file = args.project_file
+    current_project_yml = os.path.join(current_path, "project.yml")
+    if len(sys.argv) == 1 and not os.path.exists(current_project_yml):
+        answer = input(
+            f"No project.yml found in current folder.  Is it OK to generate one at {current_project_yml} for you? (y/N) "
+        )
+        if answer.strip().upper() == "Y":
+            shutil.copyfile(os.path.join(file_path, "project.yml"), current_project_yml)
+            print(f"{current_project_yml} was created.  Please edit it to fit your FL configuration.")
+        exit(0)
+
+    workspace = args.workspace
+    workspace_full_path = os.path.join(current_path, workspace)
+
+    project_full_path = os.path.join(current_path, project_file)
+    print(f"Project yaml file: {project_full_path}.")
+
+    project = yaml.load(open(project_full_path, "r"), Loader=yaml.Loader)
+    api_version = project.get("api_version")
+    if api_version not in [2]:
+        raise ValueError(f"Incompatible API version found in {project_full_path}")
+
+    study_name = project.get("name")
+    study_description = project.get("description", "")
+    participants = list()
+    for p in project.get("participants"):
+        participants.append(Participant(**p))
+    study = Study(name=study_name, description=study_description, participants=participants)
+
+    builders = list()
+    for b in project.get("builders"):
+        path = b.get("path")
+        args = b.get("args")
+        builders.append(instantiate_class(path, args))
+
+    provisioner = Provisioner(workspace_full_path, builders)
+
+    provisioner.provision(study)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/nvflare/lighter/spec.py b/nvflare/lighter/spec.py
new file mode 100644
index 0000000000..224c344ae7
--- /dev/null
+++ b/nvflare/lighter/spec.py
@@ -0,0 +1,151 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import traceback
+from abc import ABC
+from typing import List
+
+from nvflare.apis.utils.format_check import name_check
+
+"""
+
+Provisioner's tasks:
+
+- Maintain the provision workspace folder structure;
+- Invoke Builders to fill the content of each startup kit
+
+ROOT_WORKSPACE Folder Structure
+
+root_workspace_dir_name: this is the root of the workspace
+    study_dir_name: the root dir of the study, could be named after the study
+        resources: stores resource files (templates, configs, etc.) of the Provisioner and Builders
+        prod: stores the current set of startup kits (production)
+            participate_dir: stores content files generated by builders
+        wip: stores the set of startup kits to be created (WIP)
+            participate_dir: stores content files generated by builders
+        state: stores the persistent state of the Builders
+
+"""
+
+
+class Participant(object):
+    def __init__(self, type: str, name: str, org: str, enable_byoc: bool = False, *args, **kwargs):
+        err, reason = name_check(name, type)
+        if err:
+            raise ValueError(reason)
+        err, reason = name_check(org, "org")
+        if err:
+            raise ValueError(reason)
+        self.type = type
+        self.name = name
+        self.org = org
+        self.subject = name
+        self.enable_byoc = enable_byoc
+        self.props = kwargs
+
+
+class Study(object):
+    def __init__(self, name: str, description: str, participants: List[Participant]):
+        self.name = name
+        all_names = list()
+        for p in participants:
+            if p.name in all_names:
+                raise ValueError(f"Unable to add a duplicate name {p.name} into this study.")
+            else:
+                all_names.append(p.name)
+        self.description = description
+        self.participants = participants
+
+    def get_participants_by_type(self, type, first_only=True):
+        found = list()
+        for p in self.participants:
+            if p.type == type:
+                if first_only:
+                    return p
+                else:
+                    found.append(p)
+        return found
+
+
+class Builder(ABC):
+    def initialize(self, ctx: dict):
+        pass
+
+    def build(self, study: Study, ctx: dict):
+        pass
+
+    def finalize(self, ctx: dict):
+        pass
+
+    def get_wip_dir(self, ctx: dict):
+        return ctx.get("wip_dir")
+
+    def get_kit_dir(self, participate: Participant, ctx: dict):
+        return os.path.join(self.get_wip_dir(ctx), participate.name, "startup")
+
+    def get_state_dir(self, ctx: dict):
+        return ctx.get("state_dir")
+
+    def get_resources_dir(self, ctx: dict):
+        return ctx.get("resources_dir")
+
+
+class Provisioner(object):
+    def __init__(self, root_dir: str, builders: List[Builder]):
+        self.root_dir = root_dir
+        self.builders = builders
+        self.ctx = None
+
+    def _make_dir(self, dirs):
+        for dir in dirs:
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+
+    def _prepare_workspace(self, ctx):
+        workspace = ctx.get("workspace")
+        wip_dir = os.path.join(workspace, "wip")
+        state_dir = os.path.join(workspace, "state")
+        resources_dir = os.path.join(workspace, "resources")
+        ctx.update(dict(wip_dir=wip_dir, state_dir=state_dir, resources_dir=resources_dir))
+        dirs = [workspace, resources_dir, wip_dir, state_dir]
+        self._make_dir(dirs)
+
+    def provision(self, study: Study):
+        # ctx = {"workspace": os.path.join(self.root_dir, study.name), "study": study}
+        workspace = os.path.join(self.root_dir, study.name)
+        ctx = {"workspace": workspace}  # study is more static information while ctx is dynamic
+        self._prepare_workspace(ctx)
+        try:
+            for b in self.builders:
+                b.initialize(ctx)
+
+            # call builders!
+            for b in self.builders:
+                b.build(study, ctx)
+
+            for b in self.builders[::-1]:
+                b.finalize(ctx)
+
+        except BaseException as ex:
+            prod_dir = ctx.get("current_prod_dir")
+            if prod_dir:
+                shutil.rmtree(prod_dir)
+            print("Exception raised during provision.  Incomplete prod_n folder removed.")
+            traceback.print_exc()
+        finally:
+            wip_dir = ctx.get("wip_dir")
+            if wip_dir:
+                shutil.rmtree(wip_dir)
diff --git a/nvflare/lighter/utils.py b/nvflare/lighter/utils.py
new file mode 100644
index 0000000000..21989b3960
--- /dev/null
+++ b/nvflare/lighter/utils.py
@@ -0,0 +1,55 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import random
+
+import yaml
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import padding
+
+
+def generate_password():
+    s = "abcdefghijklmnopqrstuvwxyz01234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    passlen = 16
+    p = "".join(random.sample(s, passlen))
+    return p
+
+
+def sign_all(content_folder, signing_pri_key):
+    signatures = dict()
+    for f in os.listdir(content_folder):
+        path = os.path.join(content_folder, f)
+        if os.path.isfile(path):
+            signature = signing_pri_key.sign(
+                data=open(path, "rb").read(),
+                padding=padding.PSS(
+                    mgf=padding.MGF1(hashes.SHA256()),
+                    salt_length=padding.PSS.MAX_LENGTH,
+                ),
+                algorithm=hashes.SHA256(),
+            )
+            signatures[f] = signature
+    return signatures
+
+
+def load_yaml(file_name):
+    return yaml.load(open(file_name, "r"), Loader=yaml.Loader)
+
+
+def sh_replace(src, mapping_dict):
+    result = src
+    for k, v in mapping_dict.items():
+        result = result.replace("{~~" + k + "~~}", str(v))
+    return result
diff --git a/nvflare/poc/Readme.rst b/nvflare/poc/Readme.rst
new file mode 100644
index 0000000000..9cf488136d
--- /dev/null
+++ b/nvflare/poc/Readme.rst
@@ -0,0 +1,43 @@
+###############################
+How to use nvFlare POC commands
+###############################
+
+nvFlare POC commands enables to set up and run the FL training without the provision steps. The FL server communicates with the clients and admin without HTTPS encryption. There's no
+SSL certificates required on server, or the client and admin. You can run the FL server, client and admin system components on
+the same machine, or from different machines.
+
+
+#. To start FL server
+
+    $ cd poc/server
+
+    $ startup/start.sh
+
+    or
+
+    $ startup/start.sh server_host(name or IP)
+
+#. To start FL client (for each client, make a copy of the poc/client folder)
+
+    $ cd poc/site-1
+
+    $ startup/start.sh
+
+    or 
+    
+    $ startup/start.sh server_host(name or IP)
+
+    or
+    
+    $ startup/start.sh server_host(name or IP) client_name
+
+#. To start Admin (use the default user "admin/admin")
+
+    $ cd poc/admin
+
+    $ startup/fl_admin.sh
+
+    or
+
+    $ startup/fl_admin.sh server_host(name or IP)
+
diff --git a/nvflare/poc/admin/startup/fl_admin.sh b/nvflare/poc/admin/startup/fl_admin.sh
new file mode 100755
index 0000000000..3840d34b9b
--- /dev/null
+++ b/nvflare/poc/admin/startup/fl_admin.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+if [ $# -eq 0 ]
+then
+  server=localhost
+else
+  server=$1
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+echo $DIR
+mkdir -p $DIR/../transfer
+python3 -m nvflare.fuel.hci.tools.admin --host ${server} --port 8003 --prompt "> " --with_file_transfer --upload_dir=$DIR/../transfer --download_dir=$DIR/../transfer --with_shell --with_login
diff --git a/nvflare/poc/client/startup/fed_client.json b/nvflare/poc/client/startup/fed_client.json
new file mode 100644
index 0000000000..30d2c405db
--- /dev/null
+++ b/nvflare/poc/client/startup/fed_client.json
@@ -0,0 +1 @@
+{"format_version": 1, "servers": [{"name": "example_project", "service": {"target": "localhost:8002", "options": [["grpc.max_send_message_length", 2147483647], ["grpc.max_receive_message_length", 2147483647]]}}], "client": {"retry_timeout": 30, "ssl_private_key": "client.key", "ssl_cert": "client.crt", "ssl_root_cert": "rootCA.pem", "compression": "Gzip"}}
\ No newline at end of file
diff --git a/examples/hello-tf2/resources/log.config b/nvflare/poc/client/startup/log.config
similarity index 100%
rename from examples/hello-tf2/resources/log.config
rename to nvflare/poc/client/startup/log.config
diff --git a/nvflare/poc/client/startup/start.sh b/nvflare/poc/client/startup/start.sh
new file mode 100755
index 0000000000..a144258449
--- /dev/null
+++ b/nvflare/poc/client/startup/start.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+site_name=site-NNN
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# export CUDA_VISIBLE_DEVICES=
+echo "WORKSPACE set to $DIR/.."
+mkdir -p $DIR/../transfer
+if [ $# -eq 0 ] ; then
+	echo "No sever name is provided.  Default server (localhost) is used"
+	echo "Usage: start.sh <SERVER_HOST <CLIENT_NAME>>"
+	server=localhost
+elif [ $# -eq 1 ] ; then
+	server=$1
+elif [ $# -eq 2 ] ; then
+	server=$1
+	site_name=$2
+else
+	echo "Usage: start.sh <SERVER_HOST <CLIENT_NAME>>"
+	exit
+fi
+sleep 1
+$DIR/sub_start.sh $site_name $server &
diff --git a/nvflare/poc/client/startup/stop_fl.sh b/nvflare/poc/client/startup/stop_fl.sh
new file mode 100755
index 0000000000..7b22fd6932
--- /dev/null
+++ b/nvflare/poc/client/startup/stop_fl.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+echo "Shutdown request created.  FL system will shutdown soon."
+touch $DIR/../shutdown.fl
diff --git a/nvflare/poc/client/startup/sub_start.sh b/nvflare/poc/client/startup/sub_start.sh
new file mode 100755
index 0000000000..daa37e12b3
--- /dev/null
+++ b/nvflare/poc/client/startup/sub_start.sh
@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+client=$1
+server=$2
+
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# export CUDA_VISIBLE_DEVICES=
+echo "WORKSPACE set to $DIR/.."
+mkdir -p $DIR/../transfer
+export PYTHONPATH=/local/custom:$PYTHONPATH
+echo "PYTHONPATH is $PYTHONPATH"
+
+SECONDS=0
+lst=-400
+restart_count=0
+start_fl() {
+  if [[ $(( $SECONDS - $lst )) -lt 300 ]]; then
+    ((restart_count++))
+  else
+    restart_count=0
+  fi
+  if [[ $(($SECONDS - $lst )) -lt 300 && $restart_count -ge 5 ]]; then
+    echo "System is in trouble and unable to start the task!!!!!"
+    rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl
+    exit
+  fi
+  lst=$SECONDS
+((python3 -u -m nvflare.private.fed.app.client.client_train -m $DIR/.. -s fed_client.json --set secure_train=false multi_gpu=false uid=${client} cross_site_validate=false config_folder=config host=${server} 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | tee -a $DIR/../log.txt &)
+  pid=`cat $DIR/../pid.fl`
+}
+
+stop_fl() {
+  if [[ ! -f "$DIR/../pid.fl" ]]; then
+    echo "No pid.fl.  No need to kill process."
+    return
+  fi
+  pid=`cat $DIR/../pid.fl`
+  sleep 10
+  kill -0 ${pid} 2> /dev/null 1>&2
+  if [[ $? -ne 0 ]]; then
+    echo "Process alreday terminated"
+    return
+  fi
+  kill -9 $pid
+  rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl
+}
+  
+if [[ -f "$DIR/../daemon_pid.fl" ]]; then
+  dpid=`cat $DIR/../daemon_pid.fl`
+  kill -0 ${dpid} 2> /dev/null 1>&2
+  if [[ $? -eq 0 ]]; then
+    echo "There seems to be one instance, pid=$dpid, running."
+    echo "If you are sure it's not the case, please kill process $dpid."
+    exit
+  fi
+  rm -f $DIR/../daemon_pid.fl
+fi
+
+echo $BASHPID > $DIR/../daemon_pid.fl
+
+while true
+do
+  sleep 5
+  if [[ ! -f "$DIR/../pid.fl" ]]; then
+    start_fl
+    continue
+  fi
+  pid=`cat $DIR/../pid.fl`
+  kill -0 ${pid} 2> /dev/null 1>&2
+  if [[ $? -ne 0 ]]; then
+    if [[ -f "$DIR/../shutdown.fl" ]]; then
+      echo "Gracefully shutdown."
+      break
+    fi
+    start_fl
+    continue
+  fi
+  if [[ -f "$DIR/../shutdown.fl" ]]; then
+    echo "About to shutdown."
+    stop_fl
+    break
+  fi
+  if [[ -f "$DIR/../restart.fl" ]]; then
+    echo "About to restart."
+    stop_fl
+  fi
+done
+
+rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl
diff --git a/nvflare/poc/server/startup/fed_server.json b/nvflare/poc/server/startup/fed_server.json
new file mode 100644
index 0000000000..a766632b55
--- /dev/null
+++ b/nvflare/poc/server/startup/fed_server.json
@@ -0,0 +1 @@
+{"format_version": 1, "servers": [{"name": "example_project", "service": {"target": "localhost:8002", "options": [["grpc.max_send_message_length", 2147483647], ["grpc.max_receive_message_length", 2147483647]]}, "admin_host": "localhost", "admin_port": 8003, "ssl_private_key": "server.key", "ssl_cert": "server.crt", "ssl_root_cert": "rootCA.pem", "max_num_clients": 100, "heart_beat_timeout": 600, "num_server_workers": 4, "compression": "Gzip", "admin_storage": "transfer", "min_num_clients": 1}] }
\ No newline at end of file
diff --git a/nvflare/poc/server/startup/log.config b/nvflare/poc/server/startup/log.config
new file mode 100644
index 0000000000..6b9761569b
--- /dev/null
+++ b/nvflare/poc/server/startup/log.config
@@ -0,0 +1,27 @@
+[loggers]
+keys=root,modelLogger
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=fullFormatter
+
+[logger_root]
+level=INFO
+handlers=consoleHandler
+
+[logger_modelLogger]
+level=DEBUG
+handlers=consoleHandler
+qualname=modelLogger
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=fullFormatter
+args=(sys.stdout,)
+
+[formatter_fullFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
diff --git a/nvflare/poc/server/startup/start.sh b/nvflare/poc/server/startup/start.sh
new file mode 100755
index 0000000000..fc21e885f3
--- /dev/null
+++ b/nvflare/poc/server/startup/start.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export CUDA_VISIBLE_DEVICES=
+echo "WORKSPACE set to $DIR/.."
+mkdir -p $DIR/../transfer 
+$DIR/sub_start.sh $1 &
diff --git a/nvflare/poc/server/startup/stop_fl.sh b/nvflare/poc/server/startup/stop_fl.sh
new file mode 100755
index 0000000000..7b22fd6932
--- /dev/null
+++ b/nvflare/poc/server/startup/stop_fl.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+echo "Shutdown request created.  FL system will shutdown soon."
+touch $DIR/../shutdown.fl
diff --git a/nvflare/poc/server/startup/sub_start.sh b/nvflare/poc/server/startup/sub_start.sh
new file mode 100755
index 0000000000..7fd64419ae
--- /dev/null
+++ b/nvflare/poc/server/startup/sub_start.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+host=$1
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export CUDA_VISIBLE_DEVICES=
+echo "WORKSPACE set to $DIR/.."
+mkdir -p $DIR/../transfer 
+
+SECONDS=0
+lst=-400
+restart_count=0
+start_fl() {
+  if [[ $(( $SECONDS - $lst )) -lt 300 ]]; then
+    ((restart_count++))
+  else
+    restart_count=0
+  fi
+  if [[ $(($SECONDS - $lst )) -lt 300 && $restart_count -ge 5 ]]; then
+    echo "System is in trouble and unable to start the task!!!!!"
+    rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl
+    exit
+  fi
+  lst=$SECONDS
+((python3 -u -m nvflare.private.fed.app.server.server_train -m $DIR/.. -s fed_server.json --set secure_train=false config_folder=config host=${host} 2>&1 & echo $! >&3 ) 3>$DIR/../pid.fl | tee -a $DIR/../log.txt &)
+  pid=`cat $DIR/../pid.fl`
+}
+
+stop_fl() {
+  if [[ ! -f "$DIR/../pid.fl" ]]; then
+    echo "No pid.fl.  No need to kill process."
+    return
+  fi
+  pid=`cat $DIR/../pid.fl`
+  sleep 10
+  kill -0 ${pid} 2> /dev/null 1>&2
+  if [[ $? -ne 0 ]]; then
+    echo "Process alreday terminated"
+    return
+  fi
+  kill -9 $pid
+  rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl
+}
+  
+if [[ -f "$DIR/../daemon_pid.fl" ]]; then
+  dpid=`cat $DIR/../daemon_pid.fl`
+  kill -0 ${dpid} 2> /dev/null 1>&2
+  if [[ $? -eq 0 ]]; then
+    echo "There seems to be one instance, pid=$dpid, running."
+    echo "If you are sure it's not the case, please kill process $dpid."
+    exit
+  fi
+  rm -f $DIR/../daemon_pid.fl
+fi
+
+echo $BASHPID > $DIR/../daemon_pid.fl
+
+while true
+do
+  sleep 5
+  if [[ ! -f "$DIR/../pid.fl" ]]; then
+    start_fl
+    continue
+  fi
+  pid=`cat $DIR/../pid.fl`
+  kill -0 ${pid} 2> /dev/null 1>&2
+  if [[ $? -ne 0 ]]; then
+    if [[ -f "$DIR/../shutdown.fl" ]]; then
+      echo "Gracefully shutdown."
+      break
+    fi
+    start_fl
+    continue
+  fi
+  if [[ -f "$DIR/../shutdown.fl" ]]; then
+    echo "About to shutdown."
+    stop_fl
+    break
+  fi
+  if [[ -f "$DIR/../restart.fl" ]]; then
+    echo "About to restart."
+    stop_fl
+  fi
+done
+
+rm -f $DIR/../pid.fl $DIR/../shutdown.fl $DIR/../restart.fl $DIR/../daemon_pid.fl
diff --git a/nvflare/private/__init__.py b/nvflare/private/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/private/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/private/admin_defs.py b/nvflare/private/admin_defs.py
new file mode 100644
index 0000000000..f8fbd6821f
--- /dev/null
+++ b/nvflare/private/admin_defs.py
@@ -0,0 +1,70 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+
+class MsgHeader(object):
+
+    REF_MSG_ID = "_refMsgId"
+    RETURN_CODE = "_rtnCode"
+
+
+class ReturnCode(object):
+
+    OK = "_ok"
+    ERROR = "_error"
+
+
+class Message(object):
+    def __init__(self, topic: str, body):
+        self.id = str(uuid.uuid4())
+        self.topic = topic
+        self.body = body
+        self.headers = {}
+
+    def set_header(self, key, value):
+        self.headers[key] = value
+
+    def set_headers(self, headers: dict):
+        if not headers:
+            return
+
+        assert isinstance(headers, dict)
+        if len(headers) > 0:
+            self.headers.update(headers)
+
+    def get_header(self, key, default=None):
+        return self.headers.get(key, default)
+
+    def get_ref_id(self, default=None):
+        return self.get_header(MsgHeader.REF_MSG_ID, default)
+
+    def set_ref_id(self, msg_id):
+        self.set_header(MsgHeader.REF_MSG_ID, msg_id)
+
+
+def error_reply(err: str) -> Message:
+    msg = Message(topic="reply", body=err)
+    msg.set_header(MsgHeader.RETURN_CODE, ReturnCode.ERROR)
+    return msg
+
+
+def ok_reply(data=None) -> Message:
+    if data is None:
+        data = "ok"
+
+    msg = Message(topic="reply", body=data)
+    msg.set_header(MsgHeader.RETURN_CODE, ReturnCode.OK)
+    return msg
diff --git a/nvflare/private/aux_runner.py b/nvflare/private/aux_runner.py
new file mode 100644
index 0000000000..769355010c
--- /dev/null
+++ b/nvflare/private/aux_runner.py
@@ -0,0 +1,178 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from multiprocessing import Lock
+
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import ReturnCode, ReservedKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, make_reply, ReservedHeaderKey
+from nvflare.apis.signal import Signal
+
+
+class AuxRunner(FLComponent):
+
+    DATA_KEY_BULK = "bulk_data"
+    TOPIC_BULK = "__runner.bulk__"
+
+    def __init__(self):
+        FLComponent.__init__(self)
+        self.run_num = None
+        self.topic_table = {
+            self.TOPIC_BULK: self._process_bulk_requests,
+        }  # topic => handler
+        self.reg_lock = Lock()
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.run_num = fl_ctx.get_run_number()
+
+    def register_aux_message_handler(self,
+                                     topic: str,
+                                     message_handle_func):
+        """
+        Register aux message handling function with specified topics.
+
+        This method should be called by ServerEngine's register_aux_message_handler method.
+
+        Args:
+            topic: the topic to be handled by the func
+            message_handle_func: the func to handle the message. Must follow aux_message_handle_func_signature.
+
+        Returns:
+
+        Exception is raised when:
+            a handler is already registered for the topic;
+            bad topic - must be a non-empty string
+            bad message_handle_func - must be callable
+        """
+        if not isinstance(topic, str):
+            raise TypeError('topic must be str, but got {}'.format(type(topic)))
+
+        if topic == self.TOPIC_BULK:
+            raise ValueError('topic value "{}" is reserved'.format(topic))
+
+        if len(topic) <= 0:
+            raise ValueError('topic must not be empty')
+
+        if message_handle_func is None:
+            raise ValueError('message handler function is not specified')
+
+        if not callable(message_handle_func):
+            raise TypeError('specified message_handle_func is not callable')
+
+        with self.reg_lock:
+            if topic in self.topic_table:
+                raise ValueError('handler already registered for topic {}'.format(topic))
+
+            self.topic_table[topic] = message_handle_func
+
+    def _process_request(self, topic: str, request: Shareable, fl_ctx: FLContext) -> Shareable:
+        """
+        NOTE: peer_ctx props must have been set into the PEER_PROPS header of the request by Engine.
+
+        Args:
+            topic: topic of the message
+            request: message to be handled
+            fl_ctx: fl context
+
+        Returns:
+
+        """
+        handler_f = self.topic_table.get(topic, None)
+        if handler_f is None:
+            self.log_error(fl_ctx, "received unknown aux message topic {}".format(topic))
+            return make_reply(ReturnCode.TOPIC_UNKNOWN)
+
+        if not isinstance(request, Shareable):
+            self.log_error(
+                fl_ctx,
+                "received invalid aux request: expects a Shareable but got {}".format(
+                    type(request)
+                )
+            )
+            return make_reply(ReturnCode.BAD_REQUEST_DATA)
+
+        peer_props = request.get_peer_props()
+        if not peer_props:
+            self.log_error(fl_ctx, "missing peer_ctx from client")
+            return make_reply(ReturnCode.MISSING_PEER_CONTEXT)
+
+        if not isinstance(peer_props, dict):
+            self.log_error(
+                fl_ctx,
+                "bad peer_props from client: expects dict but got {}".format(type(peer_props)),
+            )
+            return make_reply(ReturnCode.BAD_PEER_CONTEXT)
+
+        peer_run_num = peer_props.get(ReservedKey.RUN_NUM)
+        if peer_run_num != self.run_num:
+            self.log_error(fl_ctx, "invalid aux msg: not for the same run_number")
+            return make_reply(ReturnCode.RUN_MISMATCH)
+
+        try:
+            reply = handler_f(topic=topic, request=request, fl_ctx=fl_ctx)
+        except BaseException:
+            self.log_exception(fl_ctx, 'processing error in message handling')
+            return make_reply(ReturnCode.HANDLER_EXCEPTION)
+
+        return reply
+
+    def dispatch(self, topic: str, request: Shareable, fl_ctx: FLContext) -> Shareable:
+        """
+        This method is to be called by the Engine when an aux message is received from peer.
+
+        NOTE: peer_ctx props must have been set into the PEER_PROPS header of the request by Engine.
+
+        Args:
+            topic:
+            request:
+            fl_ctx:
+
+        Returns:
+
+        """
+        valid_reply = self._process_request(topic, request, fl_ctx)
+        if isinstance(request, Shareable):
+            cookie_jar = request.get_cookie_jar()
+            if cookie_jar:
+                valid_reply.set_cookie_jar(cookie_jar)
+
+        valid_reply.set_peer_props(fl_ctx.get_all_public_props())
+        return valid_reply
+
+    def _process_bulk_requests(self, topic: str, request: Shareable, fl_ctx: FLContext):
+        reqs = request.get(self.DATA_KEY_BULK, None)
+        if not isinstance(reqs, list):
+            self.log_error(fl_ctx, 'invalid bulk request - missing list of requests')
+            return make_reply(ReturnCode.BAD_REQUEST_DATA)
+
+        abort_signal = fl_ctx.get_run_abort_signal()
+        for req in reqs:
+            if isinstance(abort_signal, Signal) and abort_signal.triggered:
+                break
+
+            if not isinstance(req, Shareable):
+                self.log_error(fl_ctx,
+                               'invalid request in bulk: expect Shareable but got {}'.format(type(req)))
+                continue
+            req_topic = req.get_header(ReservedHeaderKey.TOPIC, '')
+            if not req_topic:
+                self.log_error(fl_ctx,
+                               'invalid request in bulk: no topic in header')
+                continue
+
+            self._process_request(req_topic, req, fl_ctx)
+        return make_reply(ReturnCode.OK)
diff --git a/nvflare/private/defs.py b/nvflare/private/defs.py
new file mode 100644
index 0000000000..e6168785dd
--- /dev/null
+++ b/nvflare/private/defs.py
@@ -0,0 +1,90 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class SpecialTaskName(object):
+
+    TRY_AGAIN = '__try_again__'
+    END_RUN = '__end_run__'
+
+
+class TaskConstant(object):
+    WAIT_TIME = "__wait_time__"
+
+
+class EngineConstant(object):
+
+    FEDERATE_CLIENT = "federate_client"
+    FL_TOKEN = "fl_token"
+    CLIENT_TOKEN_FILE = "client_token.txt"
+    ENGINE_TASK_NAME = "engine_task_name"
+
+
+class InfoCollectorTopic(object):
+
+    SHOW_STATS = 'info.show_stats'
+    SHOW_ERRORS = 'info.show_errors'
+    RESET_ERRORS = 'info.reset_errors'
+
+
+class ComponentCallerTopic(object):
+
+    CALL_COMPONENT = 'comp_caller.call'
+
+
+class TrainingTopic(object):
+
+    START = 'train.start'
+    ABORT = 'train.abort'
+    ABORT_TASK = 'train.abort_task'
+    DELETE_RUN = 'train.delete_run'
+    DEPLOY = 'train.deploy'
+    SHUTDOWN = 'train.shutdown'
+    RESTART = 'train.restart'
+    CHECK_STATUS = 'train.check_status'
+    SET_RUN_NUMBER = 'train.set_run_number'
+
+
+class RequestHeader(object):
+
+    RUN_NUM = 'run_number'
+    APP_NAME = 'app_name'
+    CONTROL_COMMAND = 'control_command'
+    CALL_NAME = 'call_name'
+    COMPONENT_TARGET = 'component_target'
+
+
+class SysCommandTopic(object):
+
+    SYS_INFO = 'sys.info'
+    SHELL = 'sys.shell'
+
+
+class ControlCommandTopic(object):
+
+    DO_COMMAND = 'control.do_command'
+
+
+class ControlCommandName(object):
+
+    ABORT_TASK = 'abort_task'
+    END_RUN = 'end_run'
+
+
+class ClientStatusKey(object):
+
+    RUN_NUM = 'run_number'
+    CURRENT_TASK = 'current_task'
+    STATUS = 'status'
+    APP_NAME = 'app_name'
diff --git a/nvflare/private/event.py b/nvflare/private/event.py
new file mode 100644
index 0000000000..e334ef808e
--- /dev/null
+++ b/nvflare/private/event.py
@@ -0,0 +1,66 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey, EventScope
+from nvflare.apis.fl_context import FLContext
+
+# do not use underscore as key name; otherwise it cannot be removed from ctx
+_KEY_EVENT_DEPTH = '###event_depth'
+_MAX_EVENT_DEPTH = 20
+
+
+def fire_event(event: str, handlers: list, ctx: FLContext):
+    """
+    Fires the specified event and invokes the list of handlers.
+
+    Args:
+        event: the event to be fired
+        handlers: handlers to be invoked
+        ctx: context for cross-component data sharing
+
+    Returns:
+
+    """
+    event_id = str(uuid.uuid4())
+    event_data = ctx.get_prop(FLContextKey.EVENT_DATA, None)
+    event_origin = ctx.get_prop(FLContextKey.EVENT_ORIGIN, None)
+    event_scope = ctx.get_prop(FLContextKey.EVENT_SCOPE, EventScope.LOCAL)
+
+    depth = ctx.get_prop(_KEY_EVENT_DEPTH, 0)
+    if depth > _MAX_EVENT_DEPTH:
+        # too many recursive event calls
+        raise RuntimeError("Recursive event calls too deep (>{})".format(_MAX_EVENT_DEPTH))
+
+    ctx.set_prop(key=_KEY_EVENT_DEPTH, value=depth + 1, private=True, sticky=False)
+
+    if handlers:
+        for h in handlers:
+            assert isinstance(h, FLComponent)
+            try:
+                # since events could be recursive (a handler fires another event) on the same fl_ctx,
+                # we need to reset these key values into the fl_ctx
+                ctx.set_prop(key=FLContextKey.EVENT_ID, value=event_id, private=True, sticky=False)
+                ctx.set_prop(key=FLContextKey.EVENT_DATA, value=event_data, private=True, sticky=False)
+                ctx.set_prop(key=FLContextKey.EVENT_ORIGIN, value=event_origin, private=True, sticky=False)
+                ctx.set_prop(key=FLContextKey.EVENT_SCOPE, value=event_scope, private=True, sticky=False)
+                h.handle_event(event, ctx)
+            except:
+                h.log_exception(ctx,
+                                'exception when handling event "{}"'.format(event),
+                                fire_event=False)
+
+    ctx.set_prop(key=_KEY_EVENT_DEPTH, value=depth, private=True, sticky=False)
diff --git a/nvflare/private/fed/__init__.py b/nvflare/private/fed/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/private/fed/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/private/fed/app/__init__.py b/nvflare/private/fed/app/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/private/fed/app/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/private/fed/app/client/__init__.py b/nvflare/private/fed/app/client/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/private/fed/app/client/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/private/fed/app/client/client_train.py b/nvflare/private/fed/app/client/client_train.py
new file mode 100644
index 0000000000..55c07e8d62
--- /dev/null
+++ b/nvflare/private/fed/app/client/client_train.py
@@ -0,0 +1,226 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides a command line interface for a federated client """
+
+import argparse
+import os
+import shutil
+import sys
+import time
+
+from nvflare.fuel.common.excepts import ConfigError
+from nvflare.fuel.sec.audit import AuditService
+from nvflare.fuel.sec.security_content_service import LoadResult, SecurityContentService
+from nvflare.fuel.utils.argument_utils import parse_vars
+from nvflare.private.fed.app.fl_conf import FLClientStarterConfiger
+from nvflare.private.fed.client.admin import FedAdminAgent
+from nvflare.private.fed.client.admin_msg_sender import AdminMessageSender
+from nvflare.private.fed.client.client_engine import ClientEngine
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--workspace", "-m", type=str, help="WORKSPACE folder", required=True)
+
+    parser.add_argument(
+        "--fed_client", "-s", type=str, help="an aggregation server specification json file", required=True
+    )
+
+    parser.add_argument("--set", metavar="KEY=VALUE", nargs="*")
+
+    parser.add_argument("--local_rank", type=int, default=0)
+
+    args = parser.parse_args()
+    kv_list = parse_vars(args.set)
+
+    args.train_config = "config/config_train.json"
+    config_folder = kv_list.get("config_folder", "")
+    if config_folder == "":
+        args.client_config = "config_fed_client.json"
+    else:
+        args.client_config = config_folder + "/config_fed_client.json"
+    args.env = "config/environment.json"
+    args.log_config = None
+
+    try:
+        remove_restart_file(args)
+    except BaseException:
+        print("Could not remove the restart.fl / shutdown.fl file.  Please check your system before starting FL.")
+        sys.exit(-1)
+
+    rank = args.local_rank
+
+    try:
+        os.chdir(args.workspace)
+        AuditService.initialize(audit_file_name="audit.log")
+
+        if rank == 0:
+            workspace = create_workspace(args)
+        time.sleep(rank * 2)
+
+        # trainer = WorkFlowFactory().create_client_trainer(train_configs, envs)
+        conf = FLClientStarterConfiger(
+            app_root=workspace,
+            # wf_config_file_name="config_train.json",
+            client_config_file_name=args.fed_client,
+            # env_config_file_name="environment.json",
+            log_config_file_name=workspace + "/log.config",
+            kv_list=args.set,
+        )
+        conf.configure()
+
+        trainer = conf.base_deployer
+
+        security_check(trainer.secure_train, args)
+
+        federated_client = trainer.create_fed_client()
+        # federated_client.platform = conf.wf_config_data.get("platform", "PT")
+        federated_client.use_gpu = False
+        # federated_client.cross_site_validate = kv_list.get("cross_site_validate", True)
+        federated_client.config_folder = config_folder
+
+        if rank == 0:
+            federated_client.register()
+
+        if not federated_client.token:
+            print("The client could not register to server. ")
+            raise RuntimeError("Login failed.")
+
+        federated_client.start_heartbeat()
+
+        servers = [{t["name"]: t["service"]} for t in trainer.server_config]
+
+        admin_agent = create_admin_agent(
+            trainer.client_config,
+            trainer.client_name,
+            trainer.req_processors,
+            trainer.secure_train,
+            sorted(servers)[0],
+            federated_client,
+            args,
+            trainer.multi_gpu,
+            rank,
+        )
+        admin_agent.start()
+
+        trainer.close()
+
+    except ConfigError as ex:
+        print("ConfigError:", str(ex))
+    finally:
+        # shutil.rmtree(workspace)
+        pass
+
+    sys.exit(0)
+
+
+def security_check(secure_train, args):
+    # initialize the SecurityContentService.
+    # must do this before initializing other services since it may be needed by them!
+    startup = os.path.join(args.workspace, "startup")
+    SecurityContentService.initialize(content_folder=startup)
+
+    if secure_train:
+        insecure_list = secure_content_check(args)
+        if len(insecure_list):
+            print("The following files are not secure content.")
+            for item in insecure_list:
+                print(item)
+            sys.exit(1)
+    # initialize the AuditService, which is used by command processing.
+    # The Audit Service can be used in other places as well.
+    AuditService.initialize(audit_file_name="audit.log")
+
+
+def secure_content_check(args):
+    insecure_list = []
+    data, sig = SecurityContentService.load_json(args.fed_client)
+    if sig != LoadResult.OK:
+        insecure_list.append(args.fed_client)
+
+    client = data["client"]
+    content, sig = SecurityContentService.load_content(client.get("ssl_cert"))
+    if sig != LoadResult.OK:
+        insecure_list.append(client.get("ssl_cert"))
+    content, sig = SecurityContentService.load_content(client.get("ssl_private_key"))
+    if sig != LoadResult.OK:
+        insecure_list.append(client.get("ssl_private_key"))
+    content, sig = SecurityContentService.load_content(client.get("ssl_root_cert"))
+    if sig != LoadResult.OK:
+        insecure_list.append(client.get("ssl_root_cert"))
+
+    return insecure_list
+
+
+def remove_restart_file(args):
+    restart_file = os.path.join(args.workspace, "restart.fl")
+    if os.path.exists(restart_file):
+        os.remove(restart_file)
+    restart_file = os.path.join(args.workspace, "shutdown.fl")
+    if os.path.exists(restart_file):
+        os.remove(restart_file)
+
+
+def create_workspace(args):
+    kv_vars = parse_vars(args.set)
+    workspace = "/tmp/fl/" + kv_vars.get("uid")
+
+    if os.path.exists(workspace):
+        shutil.rmtree(workspace)
+    startup = os.path.join(args.workspace, "startup")
+    shutil.copytree(startup, workspace)
+
+    return workspace
+
+
+def create_admin_agent(
+    client_args, client_id, req_processors, secure_train, server_args, federated_client, args, is_multi_gpu, rank
+):
+    sender = AdminMessageSender(
+        client_name=federated_client.token,
+        root_cert=client_args["ssl_root_cert"],
+        ssl_cert=client_args["ssl_cert"],
+        private_key=client_args["ssl_private_key"],
+        server_args=server_args,
+        secure=secure_train,
+        is_multi_gpu=is_multi_gpu,
+        rank=rank,
+    )
+    admin_agent = FedAdminAgent(
+        client_name="admin_agent",
+        sender=sender,
+        app_ctx=ClientEngine(federated_client, federated_client.token, sender, args, rank),
+    )
+    admin_agent.app_ctx.set_agent(admin_agent)
+    for processor in req_processors:
+        admin_agent.register_processor(processor)
+
+    return admin_agent
+    # self.admin_agent.start()
+
+
+if __name__ == "__main__":
+    """
+    This is the main program when starting the NVFlare client process.
+    """
+    # # For MacOS, it needs to use 'spawn' for creating multi-process.
+    # if os.name == 'posix':
+    #     import multiprocessing
+    #     multiprocessing.set_start_method('spawn')
+
+    # import multiprocessing
+    # multiprocessing.set_start_method('spawn')
+
+    main()
diff --git a/nvflare/private/fed/app/client/sub_worker_process.py b/nvflare/private/fed/app/client/sub_worker_process.py
new file mode 100644
index 0000000000..35d7dd8c61
--- /dev/null
+++ b/nvflare/private/fed/app/client/sub_worker_process.py
@@ -0,0 +1,211 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import copy
+import os
+import threading
+import time
+import traceback
+from multiprocessing.connection import Listener, Client
+
+from nvflare.fuel.sec.security_content_service import SecurityContentService
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.signal import Signal
+from nvflare.apis.utils.fl_context_utils import get_serializable_data
+from nvflare.fuel.common.multi_process_executor_constants import CommunicateData, CommunicationMetaData
+from nvflare.private.fed.client.client_run_manager import ClientRunManager
+
+
+class EventRelayer(FLComponent):
+
+    def __init__(self, conn, local_rank):
+        super().__init__()
+        self.conn = conn
+        self.local_rank = local_rank
+
+        self.event_lock = threading.Lock()
+
+    def relay_event(self, run_manager, data):
+        with run_manager.new_context() as fl_ctx:
+            event_type = data[CommunicationMetaData.EVENT_TYPE]
+            fl_ctx.props.update(data[CommunicationMetaData.FL_CTX].props)
+
+            fl_ctx.set_prop(FLContextKey.EVENT_ORIGIN_SITE, CommunicateData.MULTI_PROCESS_EXECUTOR, private=True, sticky=False)
+            self.fire_event(event_type=event_type, fl_ctx=fl_ctx)
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        event_site = fl_ctx.get_prop(FLContextKey.EVENT_ORIGIN_SITE)
+
+        new_fl_ctx = FLContext()
+        new_fl_ctx.props.update(copy.deepcopy(get_serializable_data(fl_ctx).props))
+        if event_site != CommunicateData.MULTI_PROCESS_EXECUTOR:
+            with self.event_lock:
+                try:
+                    data = {CommunicationMetaData.EVENT_TYPE: event_type,
+                            CommunicationMetaData.RANK_NUMBER: self.local_rank,
+                            CommunicationMetaData.FL_CTX: new_fl_ctx
+                            }
+                    self.conn.send(data)
+
+                    return_data = self.conn.recv()
+                    # update the fl_ctx from the child process return data.
+                    fl_ctx.props.update(return_data[CommunicationMetaData.FL_CTX].props)
+                except BaseException:
+                    self.log_warning(fl_ctx, f"Failed to relay the event to parent process. Event: {event_type}",
+                                     fire_event=False)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--workspace", "-m", type=str, help="WORKSPACE folder", required=True)
+    # parser.add_argument("--parent_port", type=str, help="Parent listen port", required=True)
+    parser.add_argument("--ports", type=str, help="Listen ports", required=True)
+    # parser.add_argument("--local_rank", type=int, default=0)
+    args = parser.parse_args()
+    listen_ports = list(map(int, args.ports.split("-")))
+    # parent_port = args.parent_port
+
+    startup = os.path.join(args.workspace, "startup")
+    SecurityContentService.initialize(content_folder=startup)
+
+    # local_rank = args.local_rank
+    local_rank = int(os.environ['LOCAL_RANK'])
+    listen_port = listen_ports[local_rank * 3]
+    exe_conn = _create_connection(listen_port)
+
+    listen_port = listen_ports[local_rank * 3 + 1]
+    handle_conn = _create_connection(listen_port)
+
+    listen_port = listen_ports[local_rank * 3 + 2]
+
+    event_conn = None
+    while not event_conn:
+        try:
+            address = ("localhost", listen_port)
+            event_conn = Client(address, authkey=CommunicationMetaData.PARENT_PASSWORD.encode())
+        except Exception as e:
+            time.sleep(1.0)
+            pass
+
+    data = exe_conn.recv()
+
+    client_name = data[CommunicationMetaData.FL_CTX].get_prop(FLContextKey.CLIENT_NAME)
+    run_number = data[CommunicationMetaData.FL_CTX].get_prop(FLContextKey.CURRENT_RUN)
+    workspace = data[CommunicationMetaData.FL_CTX].get_prop(FLContextKey.WORKSPACE_OBJECT)
+    run_manager = ClientRunManager(
+        client_name=client_name,
+        run_num=int(run_number),
+        workspace=workspace,
+        client=None,
+        components=data[CommunicationMetaData.COMPONENTS],
+        handlers=data[CommunicationMetaData.HANDLERS],
+        conf=None
+    )
+
+    relayer = EventRelayer(event_conn, local_rank)
+    run_manager.add_handler(relayer)
+    run_manager.components[CommunicationMetaData.RELAYER] = relayer
+
+    executor = data[CommunicationMetaData.LOCAL_EXECUTOR]
+    exe_conn.send({CommunicationMetaData.RANK_PROCESS_STARTED: True})
+
+    exe_thread = threading.Thread(
+        target=execute, args=(run_manager, local_rank, exe_conn, executor)
+    )
+    exe_thread.start()
+
+    event_thread = threading.Thread(
+        target=handle_event, args=(run_manager, local_rank, handle_conn)
+    )
+    event_thread.start()
+
+    with run_manager.new_context() as fl_ctx:
+        fl_ctx.set_prop(FLContextKey.RANK_NUMBER, local_rank, private=True, sticky=True)
+        num_of_processes = int(len(listen_ports) / 3)
+        fl_ctx.set_prop(FLContextKey.NUM_OF_PROCESSES, num_of_processes, private=True, sticky=True)
+
+    exe_thread.join()
+    event_thread.join()
+
+
+def _create_connection(listen_port):
+    address = ("localhost", int(listen_port))
+    listener = Listener(address, authkey=CommunicationMetaData.CHILD_PASSWORD.encode())
+    conn = listener.accept()
+    return conn
+
+
+def execute(run_manager, local_rank, exe_conn, executor):
+    try:
+        abort_signal = None
+        while True:
+            data = exe_conn.recv()
+
+            command = data[CommunicationMetaData.COMMAND]
+            if command == CommunicateData.EXECUTE:
+                with run_manager.new_context() as fl_ctx:
+                    abort_signal = Signal()
+
+                    task_name = data[CommunicationMetaData.TASK_NAME]
+                    shareable = data[CommunicationMetaData.SHAREABLE]
+                    fl_ctx.props.update(data[CommunicationMetaData.FL_CTX].props)
+
+                    shareable = executor.execute(
+                        task_name=task_name,
+                        shareable=shareable,
+                        fl_ctx=fl_ctx,
+                        abort_signal=abort_signal)
+                    if local_rank == 0:
+                        return_data = {CommunicationMetaData.SHAREABLE: shareable,
+                                       CommunicationMetaData.FL_CTX: get_serializable_data(fl_ctx)}
+                        exe_conn.send(return_data)
+
+            elif command == CommunicateData.CLOSE:
+                if abort_signal:
+                    abort_signal.trigger(True)
+                break
+    except Exception:
+        traceback.print_exc()
+        print("If you abort client you can ignore this exception.")
+
+
+def handle_event(run_manager, local_rank, exe_conn):
+    try:
+        while True:
+            data = exe_conn.recv()
+
+            command = data[CommunicationMetaData.COMMAND]
+            if command == CommunicateData.HANDLE_EVENT:
+                event_relayer = run_manager.get_component(CommunicationMetaData.RELAYER)
+                event_relayer.relay_event(run_manager, data)
+
+                fl_ctx = data[CommunicationMetaData.FL_CTX]
+                if local_rank == 0:
+                    return_data = {CommunicationMetaData.FL_CTX: get_serializable_data(fl_ctx)}
+                    exe_conn.send(return_data)
+            elif command == CommunicateData.CLOSE:
+                break
+    except Exception:
+        traceback.print_exc()
+        print("If you abort client you can ignore this exception.")
+
+
+if __name__ == "__main__":
+    """
+    This is the program for running rank processes in multi-process mode.
+    """
+    main()
diff --git a/nvflare/private/fed/app/client/worker_process.py b/nvflare/private/fed/app/client/worker_process.py
new file mode 100644
index 0000000000..b0681d20af
--- /dev/null
+++ b/nvflare/private/fed/app/client/worker_process.py
@@ -0,0 +1,187 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides a command line interface for a federated client trainer"""
+
+import traceback
+
+import argparse
+import os
+import sys
+
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.workspace import Workspace
+from nvflare.fuel.sec.security_content_service import SecurityContentService
+from nvflare.fuel.utils.argument_utils import parse_vars
+from nvflare.private.defs import EngineConstant
+from nvflare.private.fed.app.fl_conf import FLClientStarterConfiger
+from nvflare.private.fed.client.client_json_config import ClientJsonConfigurator
+from nvflare.private.fed.client.client_run_manager import ClientRunManager
+from nvflare.private.fed.client.client_runner import ClientRunner
+from nvflare.private.fed.client.client_status import ClientStatus
+from nvflare.private.fed.client.command_agent import CommandAgent
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--workspace", "-m", type=str, help="WORKSPACE folder", required=True)
+
+    parser.add_argument(
+        "--fed_client", "-s", type=str, help="an aggregation server specification json file", required=True
+    )
+
+    parser.add_argument("--set", metavar="KEY=VALUE", nargs="*")
+
+    parser.add_argument("--local_rank", type=int, default=0)
+
+    args = parser.parse_args()
+    kv_list = parse_vars(args.set)
+
+    args.train_config = os.path.join("config", "config_train.json")
+    config_folder = kv_list.get("config_folder", "")
+    secure_train = kv_list.get("secure_train", True)
+    if config_folder == "":
+        args.client_config = "config_fed_client.json"
+    else:
+        args.client_config = os.path.join(config_folder, "config_fed_client.json")
+    args.config_folder = config_folder
+    args.env = os.path.join("config", "environment.json")
+    args.log_config = None
+
+    try:
+        remove_restart_file(args)
+    except BaseException:
+        print("Could not remove the restart.fl / shutdown.fl file.  Please check your system before starting FL.")
+        sys.exit(-1)
+
+    restart_file = os.path.join(args.workspace, "restart.fl")
+    if os.path.exists(restart_file):
+        os.remove(restart_file)
+
+    print("starting the client .....")
+
+    deployer = None
+    command_agent = None
+
+    startup = os.path.join(args.workspace, "startup")
+    SecurityContentService.initialize(content_folder=startup)
+
+    try:
+        token_file = os.path.join(args.workspace, EngineConstant.CLIENT_TOKEN_FILE)
+        with open(token_file, "r") as f:
+            token = f.readline().strip()
+            run_number = f.readline().strip()
+            client_name = f.readline().strip()
+            listen_port = f.readline().strip()
+            print(
+                "token is: {} run_number is: {} client_name: {} listen_port: {}".format(
+                    token, run_number, client_name, listen_port
+                )
+            )
+
+        workspace = os.path.join("/tmp/fl", client_name)  # TODO: find a way to replace "/tmp/fl"
+
+        conf = FLClientStarterConfiger(
+            app_root=workspace,
+            client_config_file_name=args.fed_client,
+            log_config_file_name=os.path.join(workspace, "log.config"),
+            kv_list=args.set,
+        )
+        conf.configure()
+
+        deployer = conf.base_deployer
+        federated_client = deployer.create_fed_client()
+        federated_client.status = ClientStatus.STARTING
+
+        federated_client.token = token
+        federated_client.client_name = client_name
+        federated_client.fl_ctx.set_prop(FLContextKey.CLIENT_NAME, client_name, private=False)
+        federated_client.fl_ctx.set_prop(EngineConstant.FL_TOKEN, token, private=False)
+        federated_client.fl_ctx.set_prop(FLContextKey.WORKSPACE_ROOT, args.workspace, private=True)
+
+        app_root = os.path.join(args.workspace, "run_" + str(run_number), "app_" + client_name)
+
+        client_config_file_name = os.path.join(app_root, args.client_config)
+        conf = ClientJsonConfigurator(
+            config_file_name=client_config_file_name,
+        )
+        conf.configure()
+
+        workspace = Workspace(args.workspace, client_name, config_folder)
+        run_manager = ClientRunManager(
+            client_name=client_name,
+            run_num=int(run_number),
+            workspace=workspace,
+            client=federated_client,
+            components=conf.runner_config.components,
+            handlers=conf.runner_config.handlers,
+            conf=conf,
+        )
+        federated_client.run_manager = run_manager
+
+        with run_manager.new_context() as fl_ctx:
+            fl_ctx.set_prop(FLContextKey.CLIENT_NAME, client_name, private=False)
+            fl_ctx.set_prop(EngineConstant.FL_TOKEN, token, private=False)
+            fl_ctx.set_prop(FLContextKey.WORKSPACE_ROOT, args.workspace, private=True)
+            fl_ctx.set_prop(FLContextKey.ARGS, args, sticky=True)
+            fl_ctx.set_prop(FLContextKey.APP_ROOT, app_root, private=True, sticky=True)
+            fl_ctx.set_prop(FLContextKey.WORKSPACE_OBJECT, workspace, private=True)
+            fl_ctx.set_prop(FLContextKey.SECURE_MODE, secure_train, private=True, sticky=True)
+
+            client_runner = ClientRunner(config=conf.runner_config, run_num=int(run_number), engine=run_manager)
+            run_manager.add_handler(client_runner)
+            fl_ctx.set_prop(FLContextKey.RUNNER, client_runner, private=True)
+
+            # # Start the thread for responding the inquire
+            # federated_client.stop_listen = False
+            # thread = threading.Thread(target=listen_command, args=[federated_client, int(listen_port), client_runner])
+            # thread.start()
+            # Start the command agent
+            command_agent = CommandAgent(federated_client, int(listen_port), client_runner)
+            command_agent.start(fl_ctx)
+
+        federated_client.status = ClientStatus.STARTED
+        client_runner.run(app_root, args)
+
+    except BaseException as e:
+        traceback.print_exc()
+        print("FL client execution exception: " + str(e))
+    finally:
+        # if federated_client:
+        #     federated_client.stop_listen = True
+        #     thread.join()
+        if command_agent:
+            command_agent.shutdown()
+        if deployer:
+            deployer.close()
+        # address = ('localhost', 6000)
+        # conn_client = Client(address, authkey='client process secret password'.encode())
+        # conn_client.send('bye')
+
+
+def remove_restart_file(args):
+    restart_file = os.path.join(args.workspace, "restart.fl")
+    if os.path.exists(restart_file):
+        os.remove(restart_file)
+    restart_file = os.path.join(args.workspace, "shutdown.fl")
+    if os.path.exists(restart_file):
+        os.remove(restart_file)
+
+
+if __name__ == "__main__":
+    """
+    This is the program when starting the child process for running the NVFlare executor.
+    """
+
+    main()
diff --git a/nvflare/private/fed/app/fl_conf.py b/nvflare/private/fed/app/fl_conf.py
new file mode 100644
index 0000000000..05411638ad
--- /dev/null
+++ b/nvflare/private/fed/app/fl_conf.py
@@ -0,0 +1,205 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import logging.config
+import os
+
+from nvflare.fuel.utils.argument_utils import parse_vars
+from nvflare.fuel.utils.json_scanner import Node
+from nvflare.fuel.utils.wfconf import ConfigContext
+from nvflare.private.fed.client.base_client_deployer import BaseClientDeployer
+from nvflare.private.json_configer import JsonConfigurator
+from .trainers.server_deployer import ServerDeployer
+
+FL_PACKAGES = ["nvflare"]
+FL_MODULES = ["server", "client", "app"]
+
+
+class FLServerStarterConfiger(JsonConfigurator):
+    def __init__(
+        self,
+        app_root: str,
+        server_config_file_name=None,
+        log_config_file_name=None,
+        kv_list=None,
+        logging_config=True,
+    ):
+
+        base_pkgs = FL_PACKAGES
+        module_names = FL_MODULES
+
+        if kv_list:
+            assert isinstance(kv_list, list), "cmd_vars must be list, but got {}".format(type(kv_list))
+            self.cmd_vars = parse_vars(kv_list)
+        else:
+            self.cmd_vars = {}
+
+        if logging_config:
+            log_config_file_path = os.path.join(app_root, log_config_file_name)
+            assert os.path.isfile(log_config_file_path), "missing log config file {}".format(log_config_file_path)
+            logging.config.fileConfig(fname=log_config_file_path, disable_existing_loggers=False)
+
+        server_config_file_name = os.path.join(app_root, server_config_file_name)
+
+        JsonConfigurator.__init__(
+            self,
+            config_file_name=server_config_file_name,
+            base_pkgs=base_pkgs,
+            module_names=module_names,
+            exclude_libs=True,
+        )
+
+        self.app_root = app_root
+        self.server_config_file_name = server_config_file_name
+
+        self.config_validator = None
+        self.enable_byoc = False
+
+    def start_config(self, config_ctx: ConfigContext):
+        super().start_config(config_ctx)
+
+        # loading server specifications
+        try:
+            for server in self.config_data["servers"]:
+                if server.get("ssl_private_key"):
+                    server["ssl_private_key"] = os.path.join(self.app_root, server["ssl_private_key"])
+                if server.get("ssl_cert"):
+                    server["ssl_cert"] = os.path.join(self.app_root, server["ssl_cert"])
+                if server.get("ssl_root_cert"):
+                    server["ssl_root_cert"] = os.path.join(self.app_root, server["ssl_root_cert"])
+        except Exception:
+            raise ValueError("Server config error: '{}'".format(self.server_config_file_name))
+
+    def process_config_element(self, config_ctx: ConfigContext, node: Node):
+        # JsonConfigurator.process_config_element(self, config_ctx, node)
+
+        element = node.element
+        path = node.path()
+
+        if path == "enable_byoc":
+            self.enable_byoc = element
+            return
+
+    def finalize_config(self, config_ctx: ConfigContext):
+        secure_train = False
+        if self.cmd_vars.get("secure_train"):
+            secure_train = self.cmd_vars["secure_train"]
+        if not secure_train:
+            self.enable_byoc = True
+
+        build_ctx = {
+            "secure_train": secure_train,
+            "app_validator": self.config_validator,
+            "server_config": self.config_data["servers"],
+            "server_host": self.cmd_vars.get("host", None),
+            "enable_byoc": self.enable_byoc
+        }
+
+        deployer = ServerDeployer()
+        deployer.build(build_ctx)
+        self.deployer = deployer
+
+
+class FLClientStarterConfiger(JsonConfigurator):
+    def __init__(
+        self,
+        app_root: str,
+        client_config_file_name=None,
+        log_config_file_name=None,
+        kv_list=None,
+        logging_config=True,
+    ):
+
+        base_pkgs = FL_PACKAGES
+        module_names = FL_MODULES
+
+        if kv_list:
+            assert isinstance(kv_list, list), "cmd_vars must be list, but got {}".format(type(kv_list))
+            self.cmd_vars = parse_vars(kv_list)
+        else:
+            self.cmd_vars = {}
+
+        if logging_config:
+            log_config_file_path = os.path.join(app_root, log_config_file_name)
+            assert os.path.isfile(log_config_file_path), "missing log config file {}".format(log_config_file_path)
+            logging.config.fileConfig(fname=log_config_file_path, disable_existing_loggers=False)
+
+        client_config_file_name = os.path.join(app_root, client_config_file_name)
+
+        JsonConfigurator.__init__(
+            self,
+            config_file_name=client_config_file_name,
+            base_pkgs=base_pkgs,
+            module_names=module_names,
+            exclude_libs=True,
+        )
+
+        self.app_root = app_root
+        self.client_config_file_name = client_config_file_name
+        self.enable_byoc = False
+
+    def process_config_element(self, config_ctx: ConfigContext, node: Node):
+        # JsonConfigurator.process_config_element(self, config_ctx, node)
+
+        element = node.element
+        path = node.path()
+
+        if path == "enable_byoc":
+            self.enable_byoc = element
+            return
+
+    def start_config(self, config_ctx: ConfigContext):
+        super().start_config(config_ctx)
+
+        try:
+            client = self.config_data["client"]
+            if client.get("ssl_private_key"):
+                client["ssl_private_key"] = os.path.join(self.app_root, client["ssl_private_key"])
+            if client.get("ssl_cert"):
+                client["ssl_cert"] = os.path.join(self.app_root, client["ssl_cert"])
+            if client.get("ssl_root_cert"):
+                client["ssl_root_cert"] = os.path.join(self.app_root, client["ssl_root_cert"])
+        except Exception:
+            raise ValueError("Client config error: '{}'".format(self.client_config_file_name))
+
+    def finalize_config(self, config_ctx: ConfigContext):
+        secure_train = False
+        if self.cmd_vars.get("secure_train"):
+            secure_train = self.cmd_vars["secure_train"]
+        if not secure_train:
+            self.enable_byoc = True
+
+        build_ctx = {
+            "client_name": self.cmd_vars.get("uid", ""),
+            "server_config": self.config_data.get("servers", []),
+            "client_config": self.config_data["client"],
+            "secure_train": secure_train,
+            "server_host": self.cmd_vars.get("host", None),
+            "enable_byoc": self.enable_byoc,
+        }
+
+        self.base_deployer = BaseClientDeployer()
+        self.base_deployer.build(build_ctx)
+        # self.trainer = trainer
+        # self.client_trainer = trainer
+
+    # def build_nested(self, config_dict, app_root):
+    #     if get_config_classname(config_dict) == "ClaraEvaluator":
+    #         config_dict["args"]["app_root"] = app_root
+    #     for k in config_dict["args"]:
+    #         element = config_dict["args"][k]
+    #         if isinstance(element, dict) and element.get("args") is not None:
+    #             config_dict["args"][k] = self.build_nested(config_dict["args"][k], app_root)
+    #     return self.build_component(config_dict)
diff --git a/nvflare/private/fed/app/server/__init__.py b/nvflare/private/fed/app/server/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/private/fed/app/server/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/private/fed/app/server/server_train.py b/nvflare/private/fed/app/server/server_train.py
new file mode 100644
index 0000000000..06587cb7ed
--- /dev/null
+++ b/nvflare/private/fed/app/server/server_train.py
@@ -0,0 +1,232 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides a command line interface for federated server"""
+
+import argparse
+import logging
+import os
+import shutil
+import sys
+
+from nvflare.fuel.common.excepts import ConfigError
+from nvflare.fuel.hci.server.authz import AuthorizationService
+from nvflare.fuel.sec.audit import AuditService
+from nvflare.fuel.sec.security_content_service import LoadResult, SecurityContentService
+from nvflare.fuel.utils.argument_utils import parse_vars
+from nvflare.private.fed.app.fl_conf import FLServerStarterConfiger
+from nvflare.private.fed.server.admin import FedAdminServer
+from nvflare.security.security import EmptyAuthorizer, FLAuthorizer
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--workspace", "-m", type=str, help="WORKSPACE folder", required=True)
+
+    parser.add_argument(
+        "--fed_server", "-s", type=str, help="an aggregation server specification json file", required=True
+    )
+
+    parser.add_argument("--set", metavar="KEY=VALUE", nargs="*")
+
+    args = parser.parse_args()
+    kv_list = parse_vars(args.set)
+
+    args.train_config = "config/config_train.json"
+    config_folder = kv_list.get("config_folder", "")
+    if config_folder == "":
+        args.server_config = "config_fed_server.json"
+    else:
+        args.server_config = config_folder + "/config_fed_server.json"
+    args.env = "config/environment.json"
+    args.config_folder = config_folder
+    logger = logging.getLogger()
+    args.log_config = None
+
+    try:
+        remove_restart_file(args)
+    except BaseException:
+        print("Could not remove the restart.fl / shutdown.fl file.  Please check your system before starting FL.")
+        sys.exit(-1)
+
+    try:
+        os.chdir(args.workspace)
+
+        create_workspace(args)  # YC: is this still useful?
+
+        # trainer = WorkFlowFactory().create_server_trainer(train_configs, envs)
+        conf = FLServerStarterConfiger(
+            app_root="/tmp/fl_server",
+            # wf_config_file_name="config_train.json",
+            server_config_file_name=args.fed_server,
+            # env_config_file_name="environment.json",
+            log_config_file_name="/tmp/fl_server/log.config",
+            kv_list=args.set,
+        )
+        log_level = os.environ.get("FL_LOG_LEVEL", "")
+        numeric_level = getattr(logging, log_level.upper(), None)
+        if isinstance(numeric_level, int):
+            logging.getLogger().setLevel(numeric_level)
+            logger.debug("loglevel debug enabled")
+            logger.info("loglevel info enabled")
+            logger.warning("loglevel warn enabled")
+            logger.error("loglevel error enabled")
+            logger.critical("loglevel critical enabled")
+        conf.configure()
+
+        deployer = conf.deployer
+        secure_train = conf.cmd_vars.get("secure_train", False)
+
+        security_check(secure_train, args)
+
+        try:
+            # Deploy the FL server
+            services = deployer.deploy(args)
+
+            first_server = sorted(conf.config_data["servers"])[0]
+            # allow command to overwrite the admin_host
+            if conf.cmd_vars.get("host", None):
+                first_server["admin_host"] = conf.cmd_vars["host"]
+            admin_server = create_admin_server(
+                services,
+                server_conf=first_server,
+                args=args,
+                secure_train=secure_train,
+                app_validator=deployer.config_validator,
+            )
+            admin_server.start()
+
+            services.platform = "PT"
+
+            services.set_admin_server(admin_server)
+        finally:
+            deployer.close()
+
+        logger.info("Server started")
+
+    except ConfigError as ex:
+        print("ConfigError:", str(ex))
+    finally:
+        # shutil.rmtree("/tmp/fl_server")
+        pass
+
+
+def security_check(secure_train, args):
+    # initialize the SecurityContentService.
+    # must do this before initializing other services since it may be needed by them!
+    startup = os.path.join(args.workspace, "startup")
+    SecurityContentService.initialize(content_folder=startup)
+
+    if secure_train:
+        insecure_list = secure_content_check(args)
+        if len(insecure_list):
+            print("The following files are not secure content.")
+            for item in insecure_list:
+                print(item)
+            sys.exit(1)
+
+    # initialize the AuditService, which is used by command processing.
+    # The Audit Service can be used in other places as well.
+    AuditService.initialize(audit_file_name="audit.log")
+    # Initialize the AuthorizationService. It is used by command authorization
+    # We use FLAuthorizer for policy processing.
+    # AuthorizationService depends on SecurityContentService to read authorization policy file.
+    if secure_train:
+        _, err = AuthorizationService.initialize(FLAuthorizer())
+    else:
+        _, err = AuthorizationService.initialize(EmptyAuthorizer())
+
+    if err:
+        print("AuthorizationService error: {}".format(err))
+        sys.exit(1)
+
+
+def secure_content_check(args):
+    insecure_list = []
+    data, sig = SecurityContentService.load_json(args.fed_server)
+    if sig != LoadResult.OK:
+        insecure_list.append(args.fed_server)
+
+    for server in data["servers"]:
+        content, sig = SecurityContentService.load_content(server.get("ssl_cert"))
+        if sig != LoadResult.OK:
+            insecure_list.append(server.get("ssl_cert"))
+        content, sig = SecurityContentService.load_content(server.get("ssl_private_key"))
+        if sig != LoadResult.OK:
+            insecure_list.append(server.get("ssl_private_key"))
+        content, sig = SecurityContentService.load_content(server.get("ssl_root_cert"))
+        if sig != LoadResult.OK:
+            insecure_list.append(server.get("ssl_root_cert"))
+
+    if "authorization.json" in SecurityContentService.security_content_manager.signature:
+        data, sig = SecurityContentService.load_json("authorization.json")
+        if sig != LoadResult.OK:
+            insecure_list.append("authorization.json")
+
+    return insecure_list
+
+
+def remove_restart_file(args):
+    restart_file = os.path.join(args.workspace, "restart.fl")
+    if os.path.exists(restart_file):
+        os.remove(restart_file)
+    restart_file = os.path.join(args.workspace, "shutdown.fl")
+    if os.path.exists(restart_file):
+        os.remove(restart_file)
+
+
+def create_admin_server(fl_server, server_conf=None, args=None, secure_train=False, app_validator=None):
+    # sai = ServerEngine(fl_server, args)
+    users = {}
+    # Create a default user admin:admin for the POC insecure use case.
+    if not secure_train:
+        users = {
+            "admin": "e7b71aa322cecc502e9454271b98feaec594da944c369facc90ac85016dc6c74c3fd99657ebd9d083a7804c3a17ddd8c655df8bcbf172be9d0207c8c9430c19be3cd846949505d283e066434175956bf45cd1d6781e63e5be4f3e23533d4d002"
+        }
+    # cmd_modules = [ValidationCommandModule()]
+
+    root_cert = server_conf["ssl_root_cert"] if secure_train else None
+    server_cert = server_conf["ssl_cert"] if secure_train else None
+    server_key = server_conf["ssl_private_key"] if secure_train else None
+    admin_server = FedAdminServer(
+        fed_admin_interface=fl_server.engine,
+        users=users,
+        cmd_modules=fl_server.cmd_modules,
+        file_upload_dir=os.path.join(args.workspace, server_conf.get("admin_storage", "tmp")),
+        file_download_dir=os.path.join(args.workspace, server_conf.get("admin_storage", "tmp")),
+        allowed_shell_cmds=None,
+        host=server_conf.get("admin_host", "localhost"),
+        port=server_conf.get("admin_port", 5005),
+        ca_cert_file_name=root_cert,
+        server_cert_file_name=server_cert,
+        server_key_file_name=server_key,
+        accepted_client_cns=None,
+        app_validator=app_validator,
+    )
+    return admin_server
+
+
+def create_workspace(args):
+    if os.path.exists("/tmp/fl_server"):
+        shutil.rmtree("/tmp/fl_server")
+    startup = os.path.join(args.workspace, "startup")
+    shutil.copytree(startup, "/tmp/fl_server")
+
+
+if __name__ == "__main__":
+    """
+    This is the main program when starting the NVFlare server process.
+    """
+
+    main()
diff --git a/nvflare/private/fed/app/trainers/__init__.py b/nvflare/private/fed/app/trainers/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/private/fed/app/trainers/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/private/fed/app/trainers/server_deployer.py b/nvflare/private/fed/app/trainers/server_deployer.py
new file mode 100644
index 0000000000..a0971b3dfe
--- /dev/null
+++ b/nvflare/private/fed/app/trainers/server_deployer.py
@@ -0,0 +1,76 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.private.fed.server.server_cmd_modules import ServerCommandModules
+from nvflare.private.fed.server.fed_server import FederatedServer
+
+
+class ServerDeployer:
+    def __init__(self):
+        # self.app_validator = None
+        self.services = None
+        self.cmd_modules = ServerCommandModules.cmd_modules
+
+    def build(self, build_ctx):
+        self.server_config = build_ctx["server_config"]
+        self.secure_train = build_ctx["secure_train"]
+        self.config_validator = build_ctx["app_validator"]
+        self.host = build_ctx["server_host"]
+        self.enable_byoc = build_ctx["enable_byoc"]
+
+    def train(self):
+        self.services = self.deploy()
+        self.start_training(self.services)
+
+    def start_training(self, services):
+        services.start()
+
+    # def create_builder(self, services):
+    #     services.set_builder(builder=self.model)
+
+    def create_fl_server(self, args, secure_train=False):
+        # We only deploy the first server right now .....
+        first_server = sorted(self.server_config)[0]
+        wait_after_min_clients = first_server.get("wait_after_min_clients", 10)
+        heart_beat_timeout = 600
+        if first_server["heart_beat_timeout"]:
+            heart_beat_timeout = first_server["heart_beat_timeout"]
+
+        if self.host:
+            target = first_server["service"].get("target", None)
+            first_server["service"]["target"] = self.host + ":" + target.split(":")[1]
+
+        services = FederatedServer(
+            project_name=first_server.get("name", ""),
+            min_num_clients=first_server.get("min_num_clients", 1),
+            max_num_clients=first_server.get("max_num_clients", 100),
+            wait_after_min_clients=wait_after_min_clients,
+            cmd_modules=self.cmd_modules,
+            heart_beat_timeout=heart_beat_timeout,
+            args=args,
+            secure_train=secure_train,
+            enable_byoc=self.enable_byoc
+        )
+        return first_server, services
+
+    def deploy(self, args):
+        first_server, services = self.create_fl_server(args, secure_train=self.secure_train)
+        services.deploy(grpc_args=first_server, secure_train=self.secure_train)
+        # self.create_builder(services)
+        print("deployed FL server trainer.")
+        return services
+
+    def close(self):
+        if self.services:
+            self.services.close()
diff --git a/nvflare/private/fed/client/__init__.py b/nvflare/private/fed/client/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/private/fed/client/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/private/fed/client/admin.py b/nvflare/private/fed/client/admin.py
new file mode 100644
index 0000000000..5cae76d978
--- /dev/null
+++ b/nvflare/private/fed/client/admin.py
@@ -0,0 +1,193 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+import traceback
+
+from nvflare.fuel.hci.server.constants import ConnProps
+from nvflare.fuel.sec.audit import Auditor, AuditService
+from nvflare.private.admin_defs import Message, error_reply, ok_reply
+
+
+class Sender(object):
+    """
+    The Sender object integrate the agent with the underline messaging system.
+    Make sure its methods are exception-proof!
+    """
+
+    def send_reply(self, reply: Message):
+        """
+        Send the reply to the requester.
+        :param reply:
+        :return:
+        """
+        pass
+
+    def retrieve_requests(self) -> [Message]:
+        """
+        Send the message to retrieve pending requests from the Server
+        :return:
+        """
+        pass
+
+    def send_result(self, message: Message):
+        """
+        Send the processor results to server.
+        :param reply:
+        :return:
+        """
+        pass
+
+
+class RequestProcessor(object):
+    """
+    The RequestProcessor is responsible for processing a request.
+    """
+
+    def get_topics(self) -> [str]:
+        """
+        Get topics that this processor will handle
+        :return: list of topics
+        """
+        pass
+
+    def process(self, req: Message, app_ctx) -> Message:
+        """
+        Called to process the specified request
+        :param req:
+        :param app_ctx:
+        :return: a reply message
+        """
+        pass
+
+
+class FedAdminAgent(object):
+    def __init__(self, client_name, sender: Sender, app_ctx, req_poll_interval=0.5, process_poll_interval=0.1):
+        assert isinstance(sender, Sender), "sender must be Sender"
+
+        auditor = AuditService.get_auditor()
+        assert isinstance(auditor, Auditor), "auditor must be Auditor but got {}".format(type(auditor))
+
+        self.name = client_name
+        self.sender = sender
+        self.auditor = auditor
+        self.app_ctx = app_ctx
+        self.req_poll_interval = req_poll_interval
+        self.process_poll_interval = process_poll_interval
+        self.processors = {}
+        self.reqs = []
+        self.req_lock = threading.Lock()
+        self.retrieve_reqs_thread = None
+        self.process_req_thread = None
+        self.asked_to_stop = False
+
+    def register_processor(self, processor: RequestProcessor):
+        assert isinstance(processor, RequestProcessor), "processor must be RequestProcessor"
+
+        topics = processor.get_topics()
+        for topic in topics:
+            assert topic not in self.processors, "duplicate processors for topic {}".format(topic)
+            self.processors[topic] = processor
+
+    def start(self):
+        if self.retrieve_reqs_thread is None:
+            self.retrieve_reqs_thread = threading.Thread(target=_start_retriever, args=(self,))
+
+        # called from the main thread
+        if not self.retrieve_reqs_thread.is_alive():
+            self.retrieve_reqs_thread.start()
+
+        if self.process_req_thread is None:
+            self.process_req_thread = threading.Thread(target=_start_processor, args=(self,))
+
+        # called from the main thread
+        if not self.process_req_thread.is_alive():
+            self.process_req_thread.start()
+
+    def _run_retriever(self):
+        while True:
+            if self.asked_to_stop:
+                break
+
+            reqs = self.sender.retrieve_requests()
+            if reqs is not None and isinstance(reqs, list):
+                with self.req_lock:
+                    self.reqs.extend(reqs)
+
+            time.sleep(self.req_poll_interval)
+
+    def _run_processor(self):
+        while True:
+            if self.asked_to_stop:
+                break
+
+            with self.req_lock:
+                if len(self.reqs) > 0:
+                    req = self.reqs.pop(0)
+                else:
+                    req = None
+
+            if req:
+                assert isinstance(req, Message), "request must be Message but got {}".format(type(req))
+                topic = req.topic
+
+                # create audit record
+                if self.auditor:
+                    user_name = req.get_header(ConnProps.USER_NAME, "")
+                    ref_event_id = req.get_header(ConnProps.EVENT_ID, "")
+                    self.auditor.add_event(user=user_name, action=topic, ref=ref_event_id)
+
+                processor = self.processors.get(topic)
+                if processor:
+                    try:
+                        reply = processor.process(req, self.app_ctx)
+                        if reply is None:
+                            # simply ack
+                            reply = ok_reply()
+                        else:
+                            assert isinstance(
+                                reply, Message
+                            ), "processor for topic {} failed to produce valid reply".format(topic)
+                    except BaseException as e:
+                        traceback.print_exc()
+                        reply = error_reply("exception_occurred")
+                else:
+                    reply = error_reply("invalid_request")
+
+                reply.set_ref_id(req.id)
+                self.sender.send_reply(reply)
+
+            time.sleep(self.process_poll_interval)
+
+    def shutdown(self):
+        """
+        To be called by the Client Engine to gracefully shutdown the agent.
+        :return:
+        """
+        self.asked_to_stop = True
+
+        if self.retrieve_reqs_thread and self.retrieve_reqs_thread.is_alive():
+            self.retrieve_reqs_thread.join()
+
+        if self.process_req_thread and self.process_req_thread.is_alive():
+            self.process_req_thread.join()
+
+
+def _start_retriever(agent: FedAdminAgent):
+    agent._run_retriever()
+
+
+def _start_processor(agent: FedAdminAgent):
+    agent._run_processor()
diff --git a/nvflare/private/fed/client/admin_commands.py b/nvflare/private/fed/client/admin_commands.py
new file mode 100644
index 0000000000..80579b8a2b
--- /dev/null
+++ b/nvflare/private/fed/client/admin_commands.py
@@ -0,0 +1,168 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.fl_constant import FLContextKey, AdminCommandNames
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, ReservedHeaderKey
+from nvflare.private.fed.client.client_status import get_status_message
+from nvflare.widgets.info_collector import InfoCollector
+from nvflare.widgets.widget import WidgetID
+
+
+class CommandProcessor(object):
+    """
+    The CommandProcessor is responsible for processing a command from parent process.
+    """
+
+    def get_command_name(self) -> str:
+        """
+        Get command name that this processor will handle
+        :return: name of the command
+        """
+        pass
+
+    def process(self, data: Shareable, fl_ctx: FLContext):
+        """
+        Called to process the specified command
+        :param data:
+        :param fl_ctx:
+        :return: command processing result
+        """
+        pass
+
+
+class CheckStatusCommand(CommandProcessor):
+    def get_command_name(self) -> str:
+        return AdminCommandNames.CHECK_STATUS
+
+    def process(self, data: Shareable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        federated_client = engine.client
+        return get_status_message(federated_client.status)
+
+
+class AbortCommand(CommandProcessor):
+    def get_command_name(self) -> str:
+        return AdminCommandNames.ABORT
+
+    def process(self, data: Shareable, fl_ctx: FLContext):
+        client_runner = fl_ctx.get_prop(FLContextKey.RUNNER)
+        return client_runner.abort()
+
+
+class AbortTaskCommand(CommandProcessor):
+    def get_command_name(self) -> str:
+        return AdminCommandNames.ABORT_TASK
+
+    def process(self, data: Shareable, fl_ctx: FLContext):
+        client_runner = fl_ctx.get_prop(FLContextKey.RUNNER)
+        if client_runner:
+            client_runner.abort_task()
+        return None
+
+
+class ShowStatsCommand(CommandProcessor):
+    def get_command_name(self) -> str:
+        return AdminCommandNames.SHOW_STATS
+
+    def process(self, data: Shareable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        collector = engine.get_widget(WidgetID.INFO_COLLECTOR)
+        if not collector:
+            result = {"error": "no info collector"}
+        else:
+            assert isinstance(collector, InfoCollector)
+
+            result = collector.get_run_stats()
+
+        if not result:
+            result = "No stats info"
+        return result
+
+
+class ShowErrorsCommand(CommandProcessor):
+    def get_command_name(self) -> str:
+        return AdminCommandNames.SHOW_ERRORS
+
+    def process(self, data: Shareable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        collector = engine.get_widget(WidgetID.INFO_COLLECTOR)
+        if not collector:
+            result = {"error": "no info collector"}
+        else:
+            assert isinstance(collector, InfoCollector)
+
+            result = collector.get_errors()
+
+        # CommandAgent is expecting data, could not be None
+        if result is None:
+            result = "No Errors"
+        return result
+
+
+class ResetErrorsCommand(CommandProcessor):
+    def get_command_name(self) -> str:
+        return AdminCommandNames.RESET_ERRORS
+
+    def process(self, data: Shareable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        engine.reset_errors()
+
+
+class AuxCommand(CommandProcessor):
+    def get_command_name(self) -> str:
+        return AdminCommandNames.AUX_COMMAND
+
+    def process(self, data: Shareable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+
+        topic = data.get_header(ReservedHeaderKey.TOPIC)
+        return engine.dispatch(topic=topic, request=data, fl_ctx=fl_ctx)
+
+
+class ByeCommand(CommandProcessor):
+    def get_command_name(self) -> str:
+        return AdminCommandNames.SHUTDOWN
+
+    def process(self, data: Shareable, fl_ctx: FLContext):
+        return None
+
+
+class AdminCommands(object):
+    """
+    AdminCommands contains all the commands for processing the commands from the parent process.
+    """
+    commands = [
+        CheckStatusCommand(),
+        AbortCommand(),
+        AbortTaskCommand(),
+        ByeCommand(),
+        ShowStatsCommand(),
+        ShowErrorsCommand(),
+        ResetErrorsCommand(),
+        AuxCommand()
+    ]
+
+    @staticmethod
+    def get_command(command_name):
+        for command in AdminCommands.commands:
+            if command_name == command.get_command_name():
+                return command
+        return None
+
+    @staticmethod
+    def register_command(command_processor: CommandProcessor):
+        assert isinstance(command_processor, CommandProcessor)
+
+        AdminCommands.commands.append(command_processor)
diff --git a/nvflare/private/fed/client/admin_msg_sender.py b/nvflare/private/fed/client/admin_msg_sender.py
new file mode 100644
index 0000000000..ed0c7d52bb
--- /dev/null
+++ b/nvflare/private/fed/client/admin_msg_sender.py
@@ -0,0 +1,154 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+from multiprocessing.dummy import Pool as ThreadPool
+
+import grpc
+
+import nvflare.private.fed.protos.admin_pb2 as admin_msg
+import nvflare.private.fed.protos.admin_pb2_grpc as admin_service
+from nvflare.private.admin_defs import Message
+from nvflare.private.fed.utils.messageproto import message_to_proto, proto_to_message
+from .admin import Sender
+
+lock = threading.Lock()
+
+
+class AdminMessageSender(Sender):
+    def __init__(
+        self,
+        client_name,
+        root_cert=None,
+        ssl_cert=None,
+        private_key=None,
+        server_args=None,
+        secure=False,
+        is_multi_gpu=False,
+        rank=0,
+    ):
+        self.client_name = client_name
+        self.root_cert = root_cert
+        self.ssl_cert = ssl_cert
+        self.private_key = private_key
+        self.secure = secure
+        self.servers = server_args
+        self.multi_gpu = is_multi_gpu
+        self.rank = rank
+
+        self.pool = ThreadPool(len(self.servers))
+
+    def send_reply(self, message: Message):
+        if self.rank == 0:
+            # self.send_client_reply(message)
+            for taskname in tuple(self.servers):
+                self.send_client_reply(message, taskname)
+
+    def send_client_reply(self, message, taskname):
+        try:
+            with self._set_up_channel(self.servers[taskname]) as channel:
+                stub = admin_service.AdminCommunicatingStub(channel)
+
+                reply = admin_msg.Reply()
+                reply.client_name = self.client_name
+                reply.message.CopyFrom(message_to_proto(message))
+                # reply.message = message_to_proto(message)
+                stub.SendReply(reply)
+        except BaseException:
+            pass
+
+    def retrieve_requests(self) -> [Message]:
+        messages = []
+        if self.rank == 0:
+            items = self.pool.map(self.retrieve_client_requests, tuple(self.servers))
+            for item in items:
+                messages.extend(item)
+
+        # if self.multi_gpu:
+        #     # with lock:
+        #     #     from mpi4py import MPI
+        #     #     comm = MPI.COMM_WORLD
+        #     #     messages = comm.bcast(messages, root=0)
+        #     with lock:
+        #         from mpi4py import MPI
+        #
+        #         comm = MPI.COMM_WORLD
+        #         size = comm.Get_size()
+        #
+        #         if self.rank == 0:
+        #             for i in range(1, size):
+        #                 comm.send(messages, dest=i, tag=i)
+        #         else:
+        #             messages = comm.recv(source=0, tag=self.rank)
+
+        return messages
+
+    def retrieve_client_requests(self, taskname):
+        try:
+            message_list = []
+            with self._set_up_channel(self.servers[taskname]) as channel:
+                stub = admin_service.AdminCommunicatingStub(channel)
+
+                client = admin_msg.Client()
+                client.client_name = self.client_name
+                messages = stub.Retrieve(client)
+                for i in messages.message:
+                    message_list.append(proto_to_message(i))
+        except Exception as e:
+            messages = None
+        return message_list
+
+    def send_result(self, message: Message):
+        if self.rank == 0:
+            # self.send_client_reply(message)
+            for taskname in tuple(self.servers):
+                try:
+                    with self._set_up_channel(self.servers[taskname]) as channel:
+                        stub = admin_service.AdminCommunicatingStub(channel)
+
+                        reply = admin_msg.Reply()
+                        reply.client_name = self.client_name
+                        reply.message.CopyFrom(message_to_proto(message))
+                        # reply.message = message_to_proto(message)
+                        stub.SendResult(reply)
+                except BaseException:
+                    pass
+
+    def _set_up_channel(self, channel_dict):
+        """
+        Connect client to the server.
+
+        :param channel_dict: grpc channel parameters
+        :return: an initialised grpc channel
+        """
+        if self.secure:
+            with open(self.root_cert, "rb") as f:
+                trusted_certs = f.read()
+            with open(self.private_key, "rb") as f:
+                private_key = f.read()
+            with open(self.ssl_cert, "rb") as f:
+                certificate_chain = f.read()
+
+            call_credentials = grpc.metadata_call_credentials(
+                lambda context, callback: callback((("x-custom-token", self.client_name),), None)
+            )
+            credentials = grpc.ssl_channel_credentials(
+                certificate_chain=certificate_chain, private_key=private_key, root_certificates=trusted_certs
+            )
+
+            composite_credentials = grpc.composite_channel_credentials(credentials, call_credentials)
+            channel = grpc.secure_channel(**channel_dict, credentials=composite_credentials)
+        else:
+            channel = grpc.insecure_channel(**channel_dict)
+        return channel
diff --git a/nvflare/private/fed/client/base_client_deployer.py b/nvflare/private/fed/client/base_client_deployer.py
new file mode 100644
index 0000000000..2b1c1621b8
--- /dev/null
+++ b/nvflare/private/fed/client/base_client_deployer.py
@@ -0,0 +1,83 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import grpc
+
+from nvflare.apis.fl_context import FLContext
+from nvflare.private.fed.client.fed_client import FederatedClient
+from .client_req_processors import ClientRequestProcessors
+
+
+class BaseClientDeployer:
+    def __init__(self):
+        self.multi_gpu = False
+        self.outbound_filters = None
+        self.inbound_filters = None
+        self.federated_client = None
+        self.model_validator = None
+        self.cross_val_participating = False
+        self.model_registry_path = None
+        self.cross_val_timeout = None
+        self.executors = None
+
+        self.req_processors = ClientRequestProcessors.request_processors
+
+    def build(self, build_ctx):
+        self.server_config = build_ctx["server_config"]
+        self.client_config = build_ctx["client_config"]
+        self.secure_train = build_ctx["secure_train"]
+        self.client_name = build_ctx["client_name"]
+        self.host = build_ctx['server_host']
+        self.enable_byoc = build_ctx["enable_byoc"]
+
+    def set_model_manager(self, model_manager):
+        self.model_manager = model_manager
+
+    def create_fed_client(self):
+        if self.host:
+            for item in self.server_config:
+                target = item["service"].get("target", None)
+                item["service"]['target'] = self.host + ':' + target.split(':')[1]
+
+        servers = [{t["name"]: t["service"]} for t in self.server_config]
+        retry_timeout = 30
+        if "retry_timeout" in self.client_config:
+            retry_timeout = self.client_config["retry_timeout"]
+
+        compression = grpc.Compression.NoCompression
+        if "Deflate" == self.client_config.get("compression"):
+            compression = grpc.Compression.Deflate
+        elif "Gzip" == self.client_config.get("compression"):
+            compression = grpc.Compression.Gzip
+
+        self.federated_client = FederatedClient(
+            client_name=str(self.client_name),
+            # We only deploy the first server right now .....
+            server_args=sorted(servers)[0],
+            client_args=self.client_config,
+            secure_train=self.secure_train,
+            retry_timeout=retry_timeout,
+            executors=self.executors,
+            compression=compression,
+            enable_byoc=self.enable_byoc
+        )
+        return self.federated_client
+
+    def finalize(self, fl_ctx: FLContext):
+        self.close()
+
+    def close(self):
+        # if self.federated_client:
+        #     self.federated_client.model_manager.close()
+        pass
diff --git a/nvflare/private/fed/client/client_aux_runner.py b/nvflare/private/fed/client/client_aux_runner.py
new file mode 100644
index 0000000000..e37de25635
--- /dev/null
+++ b/nvflare/private/fed/client/client_aux_runner.py
@@ -0,0 +1,149 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, ReservedHeaderKey, make_reply
+from nvflare.private.aux_runner import AuxRunner
+from .client_engine_executor_spec import ClientEngineExecutorSpec
+
+
+class ClientAuxRunner(AuxRunner):
+
+    """
+    Note: The ClientEngine must create a new ClientAuxRunner object for each RUN, and make sure
+    it is added as an event handler!
+
+    """
+
+    def __init__(self):
+        AuxRunner.__init__(self)
+        self.abort_signal = None
+        self.sender = None
+        self.asked_to_stop = False
+        self.engine = None
+        self.fnf_requests = []
+        self.fnf_lock = threading.Lock()
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        AuxRunner.handle_event(self, event_type, fl_ctx)
+        if event_type == EventType.START_RUN:
+            self.engine = fl_ctx.get_engine()
+            self.abort_signal = fl_ctx.get_run_abort_signal()
+            self.sender = threading.Thread(target=self._send_fnf_requests, args=())
+            self.sender.start()
+        elif event_type == EventType.END_RUN:
+            self.asked_to_stop = True
+            if self.sender.is_alive():
+                self.sender.join()
+
+    def send_aux_request(
+            self,
+            topic: str,
+            request: Shareable,
+            timeout: float,
+            fl_ctx: FLContext) -> Shareable:
+        if not isinstance(topic, str):
+            raise TypeError('invalid topic: expects str but got {}'.format(
+                type(topic)))
+
+        if not topic:
+            raise ValueError('invalid topic: must not be empty')
+
+        if topic == self.TOPIC_BULK:
+            raise ValueError('topic value "{}" is reserved'.format(topic))
+
+        if not isinstance(timeout, float):
+            raise TypeError('invalid timeout: expects float but got {}'.format(
+                type(timeout)))
+
+        if timeout < 0:
+            raise ValueError('invalid timeout value {}: must >= 0.0'.format(timeout))
+
+        if not isinstance(fl_ctx, FLContext):
+            raise TypeError('invalid fl_ctx: expects FLContext but got {}'.format(
+                type(fl_ctx)))
+
+        req_to_send = request
+        req_to_send.set_header(ReservedHeaderKey.TOPIC, topic)
+        req_to_send.set_peer_props(fl_ctx.get_all_public_props())
+
+        if timeout <= 0.0:
+            # this is fire-and-forget request
+            with self.fnf_lock:
+                self.fnf_requests.append(req_to_send)
+                return make_reply(ReturnCode.OK)
+
+        # send regular request
+        engine = fl_ctx.get_engine()
+        assert isinstance(engine, ClientEngineExecutorSpec)
+
+        reply = engine.aux_send(
+            topic=topic,
+            request=req_to_send,
+            timeout=timeout,
+            fl_ctx=fl_ctx
+        )
+
+        # check whether the RUN should be aborted
+        if not isinstance(reply, Shareable):
+            self.log_error(
+                fl_ctx,
+                'bad reply from peer: expect Shareable but got {}'.format(type(reply)))
+            return make_reply(ReturnCode.ERROR)
+
+        rc = reply.get_return_code()
+        if rc == ReturnCode.RUN_MISMATCH:
+            self.log_info(
+                fl_ctx,
+                'got RUN_MISMATCH - asked engine to abort app'
+            )
+            engine.abort_app(run_number=self.run_num, fl_ctx=fl_ctx)
+
+        return reply
+
+    def _send_fnf_requests(self):
+        topic = self.TOPIC_BULK
+        sleep_time = 0.5
+        while True:
+            time.sleep(sleep_time)
+            if self.asked_to_stop or self.abort_signal.triggered:
+                break
+
+            with self.fnf_lock:
+                if len(self.fnf_requests) <= 0:
+                    sleep_time = 1.0
+                    continue
+
+                with self.engine.new_context() as fl_ctx:
+                    bulk = Shareable()
+                    bulk.set_header(ReservedHeaderKey.TOPIC, topic)
+                    bulk.set_peer_props(fl_ctx.get_all_public_props())
+                    bulk[self.DATA_KEY_BULK] = self.fnf_requests
+                    reply = self.engine.aux_send(
+                        topic=topic,
+                        request=bulk,
+                        timeout=1.0,
+                        fl_ctx=fl_ctx
+                    )
+
+                    rc = reply.get_return_code()
+                    if rc != ReturnCode.COMMUNICATION_ERROR:
+                        # if communication error we'll retry
+                        self.fnf_requests = []
+            sleep_time = 0.5
diff --git a/nvflare/private/fed/client/client_engine.py b/nvflare/private/fed/client/client_engine.py
new file mode 100644
index 0000000000..3f5d090232
--- /dev/null
+++ b/nvflare/private/fed/client/client_engine.py
@@ -0,0 +1,311 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import re
+import shutil
+import sys
+import time
+import traceback
+from concurrent.futures import ThreadPoolExecutor
+
+from nvflare.apis.fl_constant import MachineStatus
+from nvflare.apis.shareable import Shareable
+from nvflare.fuel.hci.zip_utils import unzip_all_from_bytes
+from nvflare.private.admin_defs import Message
+from nvflare.private.defs import ClientStatusKey
+
+from .client_engine_internal_spec import ClientEngineInternalSpec
+from .client_executor import ProcessExecutor
+from .client_run_manager import ClientRunInfo
+from .client_status import ClientStatus
+
+
+class ClientEngine(ClientEngineInternalSpec):
+    """
+    ClientEngine runs in the client parent process.
+    """
+    def __init__(self, client, client_name, sender, args, rank, workers=5):
+        self.client = client
+        self.client_name = client_name
+        self.sender = sender
+        self.args = args
+        self.rank = rank
+        self.client.process = None
+        self.client_executor = ProcessExecutor(client.client_name)
+
+        self.run_number = -1
+        self.status = MachineStatus.STOPPED
+
+        assert workers >= 1, "workers must >= 1"
+        self.executor = ThreadPoolExecutor(max_workers=workers)
+
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def set_agent(self, admin_agent):
+        self.admin_agent = admin_agent
+
+    def _get_open_port(self):
+        import socket
+
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.bind(("", 0))
+        s.listen(1)
+        port = s.getsockname()[1]
+        s.close()
+        return port
+
+    def do_validate(self, req: Message):
+        self.logger.info("starting cross site validation.")
+        future = self.executor.submit(lambda p: _do_validate(*p), [self.sender, req])
+        # thread = threading.Thread(target=_do_validate, args=(self.sender, req))
+        # thread.start()
+
+        return "validate process started."
+
+    # def client_status(self):
+    #     if self.rank == 0:
+    #         self.logger.info("check client status.")
+    #         client_name = self.client.uid
+    #         token = self.client.token
+    #         message = "client name: {}".format(client_name)
+    #         message += "\ttoken: {}".format(token)
+    #
+    #         message += "\tstatus: {}".format(self.client_executor.check_status(self.client))
+    #         # if self.client.status == ClientStatus.TRAINING_STOPPED:
+    #         #     message += '\tlocal epochs: {}'.format(self.client.model_manager.fitter.num_epochs)
+    #
+    #         return message
+    #     else:
+    #         return ""
+
+    def get_engine_status(self):
+        app_name = "?"
+        if self.run_number == -1:
+            run_number = "?"
+        else:
+            run_number = str(self.run_number)
+            run_folder = os.path.join(self.args.workspace, "run_" + str(run_number))
+            app_file = os.path.join(run_folder, "fl_app.txt")
+            if os.path.exists(app_file):
+                with open(app_file, "r") as f:
+                    app_name = f.readline().strip()
+
+        result = {
+            ClientStatusKey.APP_NAME: app_name,
+            ClientStatusKey.RUN_NUM: run_number,
+            ClientStatusKey.STATUS: self.client_executor.check_status(self.client),
+        }
+        return result
+
+    def start_app(self, run_number: int) -> str:
+        status = self.client.status
+        if status == ClientStatus.STARTING or status == ClientStatus.STARTED:
+            return "Client app already started."
+
+        app_root = os.path.join(self.args.workspace, "run_" + str(run_number), "app_" + self.client.client_name)
+        if not os.path.exists(app_root):
+            return "Client app does not exist. Please deploy it before starting client."
+
+        if self.client.enable_byoc:
+            app_custom_folder = os.path.join(app_root, "custom")
+            try:
+                sys.path.index(app_custom_folder)
+            except ValueError:
+                self.remove_custom_path()
+                sys.path.append(app_custom_folder)
+        else:
+            app_custom_folder = ""
+
+        self.logger.info("Starting client app. rank: {}".format(self.rank))
+
+        open_port = self._get_open_port()
+        self._write_token_file(run_number, open_port)
+        self.run_number = run_number
+
+        self.client_executor.start_train(self.client, self.args, app_root, app_custom_folder, open_port)
+
+        return "Start the client app..."
+
+    def set_run_number(self, run_number: int) -> str:
+        self.run_number = run_number
+        return ""
+
+    def get_client_name(self):
+        return self.client.client_name
+
+    # def wait_training_process_finish(self):
+    #     self.client.process.join()
+    #
+    #     # _cross_validation(self.client, self.args)
+    #     self.client.status = ClientStatus.STOPPED
+
+    # def start_mgpu_client(self, run_number, gpu_number):
+    #     status = self.client.status
+    #     if status == ClientStatus.STARTING or status == ClientStatus.STARTED:
+    #         return "Client already in training."
+    #
+    #     app_root = os.path.join(self.args.app, "run_" + str(run_number), "app_" + self.client.uid)
+    #     if not os.path.exists(app_root):
+    #         return "Client app does not exist. Please deploy it before start client."
+    #
+    #     app_custom_folder = os.path.join(app_root, "custom")
+    #     try:
+    #         sys.path.index(app_custom_folder)
+    #     except ValueError:
+    #         self.remove_custom_path()
+    #         sys.path.append(app_custom_folder)
+    #
+    #     self.logger.info("Starting client training. rank: {}".format(self.rank))
+    #
+    #     open_port = self._get_open_port()
+    #     self._write_token_file(run_number, open_port)
+    #
+    #     self.client_executor.start_mgpu_train(
+    #         self.client, self.args, app_root, gpu_number, app_custom_folder, open_port
+    #     )
+    #
+    #     return "Start the client..."
+
+    def _write_token_file(self, run_number, open_port):
+        token_file = os.path.join(self.args.workspace, "client_token.txt")
+        if os.path.exists(token_file):
+            os.remove(token_file)
+        with open(token_file, "wt") as f:
+            f.write("%s\n%s\n%s\n%s\n" % (self.client.token, run_number, self.client.client_name, open_port))
+
+    def wait_process_complete(self):
+        self.client.process.wait()
+
+        # self.client.cross_validation()
+        self.client.status = ClientStatus.STOPPED
+
+    def remove_custom_path(self):
+        regex = re.compile(".*/run_.*/custom")
+        custom_paths = list(filter(regex.search, sys.path))
+        for path in custom_paths:
+            sys.path.remove(path)
+
+    def abort_app(self, run_number: int) -> str:
+        status = self.client.status
+        if status == ClientStatus.STOPPED:
+            return "Client app already stopped."
+
+        if status == ClientStatus.NOT_STARTED:
+            return "Client app has not started."
+
+        if status == ClientStatus.STARTING:
+            return "Client app is starting, please wait for client to have started before abort."
+
+        self.client_executor.abort_train(self.client)
+        # self.run_number = -1
+
+        return "Abort signal has been sent to the client App."
+
+    def abort_task(self, run_number: int) -> str:
+        status = self.client.status
+        if status == ClientStatus.NOT_STARTED:
+            return "Client app has not started."
+
+        if status == ClientStatus.STARTING:
+            return "Client app is starting, please wait for started before abort_task."
+
+        self.client_executor.abort_task(self.client)
+        # self.run_number = -1
+
+        return "Abort signal has been sent to the current task. "
+
+    def shutdown(self) -> str:
+        self.logger.info("Client shutdown...")
+        touch_file = os.path.join(self.args.workspace, "shutdown.fl")
+        self.client_executor.close()
+        future = self.executor.submit(lambda p: _shutdown_client(*p), [self.client, self.admin_agent, touch_file])
+
+        return "Shutdown the client..."
+
+    def restart(self) -> str:
+        self.logger.info("Client shutdown...")
+        touch_file = os.path.join(self.args.workspace, "restart.fl")
+        self.client_executor.close()
+        future = self.executor.submit(lambda p: _shutdown_client(*p), [self.client, self.admin_agent, touch_file])
+
+        return "Restart the client..."
+
+    def deploy_app(self, app_name: str, run_num: int, client_name: str, app_data) -> str:
+        # if not os.path.exists('/tmp/tmp'):
+        #     os.makedirs('/tmp/tmp')
+        dest = os.path.join(self.args.workspace, "run_" + str(run_num), "app_" + client_name)
+        # Remove the previous deployed app.
+        if os.path.exists(dest):
+            shutil.rmtree(dest)
+
+        if not os.path.exists(dest):
+            os.makedirs(dest)
+        unzip_all_from_bytes(app_data, dest)
+
+        app_file = os.path.join(self.args.workspace, "run_" + str(run_num), "fl_app.txt")
+        if os.path.exists(app_file):
+            os.remove(app_file)
+        with open(app_file, "wt") as f:
+            f.write(f"{app_name}")
+
+        return ""
+
+    def delete_run(self, run_num: int) -> str:
+        run_number_folder = os.path.join(self.args.workspace, "run_" + str(run_num))
+        if os.path.exists(run_number_folder):
+            shutil.rmtree(run_number_folder)
+        return "Delete run folder: {}".format(run_number_folder)
+
+    def get_current_run_info(self) -> ClientRunInfo:
+        return self.client_executor.get_run_info()
+
+    def get_errors(self):
+        return self.client_executor.get_errors()
+
+    def reset_errors(self):
+        self.client_executor.reset_errors()
+
+    def send_aux_command(self, shareable: Shareable):
+        return self.client_executor.send_aux_command(shareable)
+
+
+def _do_validate(sender, message):
+    print("starting the validate process .....")
+    time.sleep(60)
+    print("Generating processing result ......")
+    reply = Message(topic=message.topic, body="")
+    sender.send_result(reply)
+    pass
+
+
+def _shutdown_client(client, admin_agent, touch_file):
+    with open(touch_file, "a"):
+        os.utime(touch_file, None)
+
+    try:
+        print("About to shutdown the client...")
+        client.communicator.heartbeat_done = True
+        time.sleep(3)
+        client.close()
+
+        if client.process:
+            client.process.terminate()
+
+        admin_agent.shutdown()
+    except BaseException as e:
+        traceback.print_exc()
+        print("FL client execution exception: " + str(e))
+        # client.status = ClientStatus.TRAINING_EXCEPTION
diff --git a/nvflare/private/fed/client/client_engine_executor_spec.py b/nvflare/private/fed/client/client_engine_executor_spec.py
new file mode 100644
index 0000000000..6d3fb07c30
--- /dev/null
+++ b/nvflare/private/fed/client/client_engine_executor_spec.py
@@ -0,0 +1,66 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.client_engine_spec import ClientEngineSpec
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.fl_context import FLContext
+
+
+class ClientEngineExecutorSpec(ClientEngineSpec):
+    """
+    The ClientEngineExecutorSpec defines the ClientEngine APIs running in the child process.
+    """
+
+    def aux_send(self,
+                 topic: str,
+                 request: Shareable,
+                 timeout: float,
+                 fl_ctx: FLContext) -> Shareable:
+        """
+        Send the request to the Server.
+
+        If reply is received, make sure to set peer_ctx into the reply shareable!
+
+        Args:
+            topic: topic of the request
+            request: request to be sent
+            timeout: number of secs to wait for reply. 0 means fire-and-forget.
+            fl_ctx: fl context
+
+        Returns: a reply.
+
+        """
+        pass
+
+    def build_component(self, config_dict):
+        """
+        Build a component from the config_dict.
+        Args:
+            config_dict:
+
+        Returns:
+
+        """
+
+    def abort_app(self, run_number: int, fl_ctx: FLContext):
+        """
+        Abort the running FL App on the client.
+        Args:
+            run_number: current_rumbre
+            fl_ctx: FLContext
+
+        Returns:
+
+        """
+        pass
diff --git a/nvflare/private/fed/client/client_engine_internal_spec.py b/nvflare/private/fed/client/client_engine_internal_spec.py
new file mode 100644
index 0000000000..e9e40bd8da
--- /dev/null
+++ b/nvflare/private/fed/client/client_engine_internal_spec.py
@@ -0,0 +1,145 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.client_engine_spec import ClientEngineSpec
+
+
+class ClientEngineInternalSpec(ClientEngineSpec):
+    """
+    The ClientEngineInternalSpec defines the ClientEngine APIs running in the parent process.
+    """
+    def get_engine_status(self):
+        pass
+
+    # def get_current_run_info(self) -> ClientRunInfo:
+    #     """
+    #     Return info of the current run.
+    #
+    #     Returns: current run info, or None if app not running.
+    #
+    #     """
+    #     pass
+
+    def get_client_name(self) -> str:
+        """
+        Get the ClientEngine client_name
+        Returns: the client_name
+
+        """
+        pass
+
+    def deploy_app(self, app_name: str, run_num: int, client_name: str, app_data) -> str:
+        """
+        Deploy the app to specified run
+
+        Args:
+            app_name: FL_app name
+            run_num: run that the app is to be deployed to
+            client_name: name of the client
+            app_data: zip data of the app
+
+        Returns: error if any
+
+        """
+        pass
+
+    def start_app(self, run_number: int) -> str:
+        """
+        Start the app for the specified run
+
+        Args:
+            run_number:
+
+        Returns: error if any
+
+        """
+        pass
+
+    def abort_app(self, run_number: int) -> str:
+        """
+        Abort the app execution in current run.
+
+        Returns: error if any
+
+        """
+        pass
+
+    def abort_task(self, run_number: int) -> str:
+        """
+        Abort the client current executing task.
+
+        Returns: error if any
+
+        """
+        pass
+
+    def delete_run(self, run_num: int) -> str:
+        """
+        Delete the specified run
+
+        Args:
+            run_num:
+
+        Returns: error if any
+
+        """
+        pass
+
+    def shutdown(self) -> str:
+        """
+        Shutdown the FL client
+
+        Returns: error if any
+
+        """
+        pass
+
+    def restart(self) -> str:
+        """
+        Restart the FL client.
+
+        Returns: error if any
+
+        """
+        pass
+
+    def set_run_number(self, run_num: int) -> str:
+        """
+        Set the client run_number
+
+        Returns: error if any
+
+        """
+        pass
+
+    # def aux_send(self,
+    #              topic: str,
+    #              request: Shareable,
+    #              timeout: float,
+    #              fl_ctx: FLContext) -> Shareable:
+    #     """
+    #     Send the request to the Server.
+    #
+    #     If reply is received, make sure to set peer_ctx into the reply shareable!
+    #
+    #     Args:
+    #         topic: topic of the request
+    #         request: request to be sent
+    #         timeout: number of secs to wait for reply. 0 means fire-and-forget.
+    #         fl_ctx: fl context
+    #
+    #     Returns: a reply.
+    #
+    #     """
+    #     pass
diff --git a/nvflare/private/fed/client/client_executor.py b/nvflare/private/fed/client/client_executor.py
new file mode 100644
index 0000000000..45dbdb673e
--- /dev/null
+++ b/nvflare/private/fed/client/client_executor.py
@@ -0,0 +1,472 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import math
+import os
+import shlex
+import subprocess
+import sys
+import threading
+import time
+from multiprocessing.connection import Client
+
+from nvflare.apis.fl_constant import AdminCommandNames, ReturnCode
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.fuel.utils.pipe.file_pipe import FilePipe
+from .client_status import ClientStatus, get_status_message
+
+
+class ClientExecutor(object):
+    def __init__(self, uid) -> None:
+        pipe_path = "/tmp/fl/" + uid + "/comm"
+        if not os.path.exists(pipe_path):
+            os.makedirs(pipe_path)
+
+        self.pipe = FilePipe(root_path=pipe_path, name="training")
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def start_train(self, client, args, app_root, app_custom_folder, listen_port):
+        """
+        start_train method to start the FL client training.
+        :param client: the FL client object.
+        :param args: admin command arguments for starting the FL client training.
+        :param app_root: the root folder of the running APP.
+        :return:
+        """
+        pass
+
+    def start_mgpu_train(self, client, args, app_root, gpu_number, app_custom_folder, listen_port):
+        """
+        start the FL client training using multi-GPU.
+        :param client: the FL client object.
+        :param args: admin command arguments for starting the FL client training.
+        :param app_root: the root folder of the running APP.
+        :param gpu_number: number of GPUs to run FL training
+        :return:
+        """
+        pass
+
+    def check_status(self, client):
+        """
+        check the status of the running client.
+        :param client: the FL client object.
+        :return: running FL client status message.
+        """
+        pass
+
+    def abort_train(self, client):
+        """
+        To abort the running client.
+        :param client: the FL client object.
+        :return: N/A
+        """
+        pass
+
+    def abort_task(self, client):
+        """
+        To abort the client executing task.
+        :param client: the FL client object.
+        :return: N/A
+        """
+        pass
+
+    def get_run_info(self):
+        """
+        To get the run_info from the InfoCollector.
+        Returns:
+
+        """
+        pass
+
+    def get_errors(self):
+        """
+        To get the error_info from the InfoCollector.
+        Returns:
+
+        """
+        pass
+
+    def reset_errors(self):
+        """
+        To reset the error_info for the InfoCollector.
+        Returns:
+
+        """
+        pass
+
+    def send_aux_command(self, shareable: Shareable):
+        """
+        To send the aux command to child process.
+        Returns:
+
+        """
+        pass
+
+    def cleanup(self):
+        self.pipe.clear()
+
+
+class ProcessExecutor(ClientExecutor):
+    """
+    Run the Client executor in a child process.
+    """
+    def __init__(self, uid):
+        ClientExecutor.__init__(self, uid)
+        # self.client = client
+
+        self.conn_client = None
+        # self.pool = None
+
+        self.listen_port = 6000
+
+        self.lock = threading.Lock()
+
+    def get_conn_client(self):
+        if not self.conn_client:
+            try:
+                address = ("localhost", self.listen_port)
+                self.conn_client = Client(address, authkey="client process secret password".encode())
+            except Exception as e:
+                pass
+
+    def create_pipe(self):
+        """Create pipe to communicate between child (training) and main (logic) thread."""
+        pipe = FilePipe(root_path="/fl/server", name="training")
+
+        return pipe
+
+    def start_train(self, client, args, app_root, app_custom_folder, listen_port):
+        # self.pool = multiprocessing.Pool(processes=1)
+        # result = self.pool.apply_async(_start_client, (client, args, app_root))
+
+        # self.conn_client, child_conn = mp.Pipe()
+        # process = multiprocessing.Process(target=_start_client, args=(client, args, app_root, child_conn, self.pipe))
+        # # process = multiprocessing.Process(target=_start_new)
+        # process.start()
+
+        self.listen_port = listen_port
+
+        new_env = os.environ.copy()
+        if app_custom_folder != "":
+            new_env["PYTHONPATH"] = new_env["PYTHONPATH"] + ":" + app_custom_folder
+
+        # self.retrieve_cross_validate_setting(client, app_root)
+
+        command_options = ""
+        for t in args.set:
+            command_options += " " + t
+        command = (
+            f"{sys.executable} -m nvflare.private.fed.app.client.worker_process -m "
+            + args.workspace
+            + " -s fed_client.json "
+            " --set" + command_options + " print_conf=True"
+        )
+        # use os.setsid to create new process group ID
+        process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=new_env)
+
+        print("training child process ID: {}".format(process.pid))
+
+        client.process = process
+        client.multi_gpu = False
+
+        client.status = ClientStatus.STARTED
+        thread = threading.Thread(
+            target=self.wait_training_process_finish, args=(client, args, app_root, app_custom_folder)
+        )
+        thread.start()
+
+    # def retrieve_cross_validate_setting(self, client, app_root):
+    #     if client.config_folder == "":
+    #         client_config = "config_fed_client.json"
+    #     else:
+    #         client_config = client.config_folder + "/config_fed_client.json"
+    #     client_config = os.path.join(app_root, client_config)
+    #     conf = Configurator(
+    #         app_root=app_root,
+    #         cmd_vars={},
+    #         env_config={},
+    #         wf_config_file_name=client_config,
+    #         base_pkgs=[],
+    #         module_names=[],
+    #     )
+    #     conf.configure()
+    #     client.cross_site_validate = conf.wf_config_data.get("cross_validate", False)
+
+    # def start_mgpu_train(self, client, args, app_root, gpu_number, app_custom_folder, listen_port):
+    #     self.listen_port = listen_port
+    #
+    #     new_env = os.environ.copy()
+    #     new_env["PYTHONPATH"] = new_env["PYTHONPATH"] + ":" + app_custom_folder
+    #
+    #     # self.retrieve_cross_validate_setting(client, app_root)
+    #
+    #     if client.platform == "PT":
+    #         command = (
+    #             f"{sys.executable} -m torch.distributed.launch --nproc_per_node="
+    #             + str(gpu_number)
+    #             + " --nnodes=1 --node_rank=0 "
+    #             + '--master_addr="localhost" --master_port=1234 '
+    #             + "-m nvflare.private.fed.app.client.worker_process -m "
+    #             + args.workspace
+    #             + " -s fed_client.json "
+    #             " --set secure_train="
+    #             + str(client.secure_train)
+    #             + " print_conf=True use_gpu=True multi_gpu=True uid="
+    #             + client.client_name
+    #             + " config_folder="
+    #             + client.config_folder
+    #         )
+    #         # use os.setsid to create new process group ID
+    #         process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=new_env)
+    #     else:
+    #         command = (
+    #             "mpirun -np "
+    #             + str(gpu_number)
+    #             + " -H localhost:"
+    #             + str(gpu_number)
+    #             + " -bind-to none -map-by slot -x NCCL_DEBUG=DEBUG -x LD_LIBRARY_PATH -x PATH "
+    #             "-mca pml ob1 -mca btl ^openib --allow-run-as-root "
+    #             f"{sys.executable} -u -m nvmidl.apps.fed_learn.client.worker_process -m "
+    #             + args.workspace
+    #             + " -s fed_client.json --set secure_train="
+    #             + str(client.secure_train)
+    #             + " multi_gpu=true uid="
+    #             + client.client_name
+    #             + " config_folder="
+    #             + client.config_folder
+    #         )
+    #         process = subprocess.Popen(shlex.split(command, " "), env=new_env)
+    #     client.process = process
+    #     client.multi_gpu = True
+    #     # self.pool = multiprocessing.Pool(processes=1)
+    #     # result = self.pool.apply_async(self.call_mpirun, (client, args, app_root))
+    #
+    #     client.status = ClientStatus.STARTED
+    #     thread = threading.Thread(
+    #         target=self.wait_training_process_finish, args=(client, args, app_root, app_custom_folder)
+    #     )
+    #     thread.start()
+
+    def check_status(self, client):
+        try:
+            self.get_conn_client()
+
+            if self.conn_client:
+                data = {"command": AdminCommandNames.CHECK_STATUS, "data": {}}
+                self.conn_client.send(data)
+                status_message = self.conn_client.recv()
+                print("check status from process listener......")
+                return status_message
+            else:
+                return get_status_message(client.status)
+        except:
+            self.logger.error("Check_status execution exception.")
+            return "execution exception. Please try again."
+
+    def get_run_info(self):
+        try:
+            self.get_conn_client()
+
+            if self.conn_client:
+                data = {"command": AdminCommandNames.SHOW_STATS, "data": {}}
+                self.conn_client.send(data)
+                run_info = self.conn_client.recv()
+                return run_info
+            else:
+                return {}
+        except:
+            self.logger.error("get_run_info() execution exception.")
+            return {"error": "no info collector. Please try again."}
+
+    def get_errors(self):
+        try:
+            self.get_conn_client()
+
+            if self.conn_client:
+                data = {"command": AdminCommandNames.SHOW_ERRORS, "data": {}}
+                self.conn_client.send(data)
+                errors_info = self.conn_client.recv()
+                return errors_info
+            else:
+                return None
+        except:
+            self.logger.error("get_errors() execution exception.")
+            return None
+
+    def reset_errors(self):
+        try:
+            self.get_conn_client()
+
+            if self.conn_client:
+                data = {"command": AdminCommandNames.RESET_ERRORS, "data": {}}
+                self.conn_client.send(data)
+        except:
+            self.logger.error("reset_errors() execution exception.")
+
+    def send_aux_command(self, shareable: Shareable):
+        try:
+            self.get_conn_client()
+            if self.conn_client:
+                data = {"command": AdminCommandNames.AUX_COMMAND, "data": shareable}
+                self.conn_client.send(data)
+                reply = self.conn_client.recv()
+                return reply
+            else:
+                return make_reply(ReturnCode.EXECUTION_EXCEPTION)
+        except:
+            return make_reply(ReturnCode.EXECUTION_EXCEPTION)
+
+    def abort_train(self, client):
+        # if client.status == ClientStatus.CROSS_SITE_VALIDATION:
+        #     # Only aborts cross site validation.
+        #     client.abort()
+        # elif client.status == ClientStatus.TRAINING_STARTED:
+        if client.status == ClientStatus.STARTED:
+            with self.lock:
+                if client.process:
+                    # if client.platform == 'PT' and client.multi_gpu:
+                    #     # kill the sub-process group directly
+                    #     os.killpg(os.getpgid(client.process.pid), 9)
+                    # else:
+                    #     client.process.terminate()
+
+                    # kill the sub-process group directly
+                    if self.conn_client:
+                        data = {"command": AdminCommandNames.ABORT, "data": {}}
+                        self.conn_client.send(data)
+                        self.logger.debug("abort sent")
+                        # wait for client to handle abort
+                        time.sleep(2.0)
+
+                    # kill the sub-process group directly
+                    try:
+                        os.killpg(os.getpgid(client.process.pid), 9)
+                        self.logger.debug("kill signal sent")
+                    except Exception as e:
+                        pass
+                    client.process.terminate()
+                    self.logger.debug("terminated")
+
+                # if self.pool:
+                #     self.pool.terminate()
+                if self.conn_client:
+                    self.conn_client.close()
+                self.conn_client = None
+                self.cleanup()
+
+        self.logger.info("Client training was terminated.")
+
+    def abort_task(self, client):
+        if client.status == ClientStatus.STARTED:
+            if self.conn_client:
+                data = {"command": AdminCommandNames.ABORT_TASK, "data": {}}
+                self.conn_client.send(data)
+                self.logger.debug("abort_task sent")
+
+    def wait_training_process_finish(self, client, args, app_root, app_custom_folder):
+        # wait for the listen_command thread to start, and send "start" message to wake up the connection.
+        start = time.time()
+        while True:
+            self.get_conn_client()
+            if self.conn_client:
+                data = {"command": AdminCommandNames.START_APP, "data": {}}
+                self.conn_client.send(data)
+                break
+            time.sleep(1.0)
+            if time.time() - start > 15:
+                break
+
+        self.logger.info("waiting for process to finish")
+        client.process.wait()
+        returncode = client.process.returncode
+        self.logger.info(f"process finished with execution code: {returncode}")
+
+        with self.lock:
+            client.process = None
+
+            if self.conn_client:
+                self.conn_client.close()
+            self.conn_client = None
+
+        # # result.get()
+        # self.pool.close()
+        # self.pool.join()
+        # self.pool.terminate()
+
+        # Not to run cross_validation in a new process any more.
+        client.cross_site_validate = False
+
+        client.status = ClientStatus.STOPPED
+
+    def close(self):
+        if self.conn_client:
+            data = {"command": AdminCommandNames.SHUTDOWN, "data": {}}
+            self.conn_client.send(data)
+            self.conn_client = None
+        self.cleanup()
+
+
+# class ThreadExecutor(ClientExecutor):
+#     def __init__(self, client, executor):
+#         self.client = client
+#         self.executor = executor
+
+#     def start_train(self, client, args, app_root, app_custom_folder, listen_port):
+#         future = self.executor.submit(lambda p: _start_client(*p), [client, args, app_root])
+
+#     def start_mgpu_train(self, client, args, app_root, gpu_number, app_custom_folder, listen_port):
+#         self.start_train(client, args, app_root)
+
+#     def check_status(self, client):
+#         return get_status_message(self.client.status)
+
+#     def abort_train(self, client):
+#         self.client.train_end = True
+#         self.client.fitter.train_ctx.ask_to_stop_immediately()
+#         self.client.fitter.train_ctx.set_prop("early_end", True)
+#         # self.client.model_manager.close()
+#         # self.client.status = ClientStatus.TRAINING_STOPPED
+#         return "Aborting the client..."
+
+
+def update_client_properties(client, trainer):
+    # servers = [{t['name']: t['service']} for t in trainer.server_config]
+    retry_timeout = 30
+    # if trainer.client_config['retry_timeout']:
+    #     retry_timeout = trainer.client_config['retry_timeout']
+    client.client_args = trainer.client_config
+    # client.servers = sorted(servers)[0]
+    # client.model_manager.federated_meta = {task_name: list() for task_name in tuple(client.servers)}
+    exclude_vars = trainer.client_config.get("exclude_vars", "dummy")
+    # client.model_manager.exclude_vars = re.compile(exclude_vars) if exclude_vars else None
+    # client.model_manager.privacy_policy = trainer.privacy
+    # client.model_manager.model_reader_writer = trainer.model_reader_writer
+    # client.model_manager.model_validator = trainer.model_validator
+    # client.pool = ThreadPool(len(client.servers))
+    # client.communicator.ssl_args = trainer.client_config
+    # client.communicator.secure_train = trainer.secure_train
+    # client.communicator.model_manager = client.model_manager
+    client.communicator.should_stop = False
+    client.communicator.retry = int(math.ceil(float(retry_timeout) / 5))
+    # client.communicator.outbound_filters = trainer.outbound_filters
+    # client.communicator.inbound_filters = trainer.inbound_filters
+    client.handlers = trainer.handlers
+    # client.inbound_filters = trainer.inbound_filters
+    client.executors = trainer.executors
+    # client.task_inbound_filters = trainer.task_inbound_filters
+    # client.task_outbound_filters = trainer.task_outbound_filters
+    # client.secure_train = trainer.secure_train
+    client.heartbeat_done = False
+    # client.fl_ctx = FLContext()
diff --git a/nvflare/private/fed/client/client_json_config.py b/nvflare/private/fed/client/client_json_config.py
new file mode 100644
index 0000000000..897b41d016
--- /dev/null
+++ b/nvflare/private/fed/client/client_json_config.py
@@ -0,0 +1,113 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_component import FLComponent
+from nvflare.fuel.utils.json_scanner import Node
+from nvflare.private.fed_json_config import FedJsonConfigurator
+from nvflare.private.json_configer import ConfigContext, ConfigError
+from .client_runner import ClientRunnerConfig
+
+
+class _ExecutorDef(object):
+
+    def __init__(self):
+        self.tasks = []
+        self.executor = None
+
+
+FL_PACKAGES = ["nvflare"]
+FL_MODULES = ["server", "client", "aggregators", "handlers", "pt", "app", "app_common", "workflows"]
+
+
+class ClientJsonConfigurator(FedJsonConfigurator):
+    def __init__(
+        self,
+        config_file_name: str,
+        exclude_libs=True
+    ):
+        base_pkgs = FL_PACKAGES
+        module_names = FL_MODULES
+
+        FedJsonConfigurator.__init__(
+            self,
+            config_file_name=config_file_name,
+            base_pkgs=base_pkgs,
+            module_names=module_names,
+            exclude_libs=exclude_libs)
+
+        self.runner_config = None
+        self.executors = []
+        self.current_exe = None
+
+    def process_config_element(self, config_ctx: ConfigContext, node: Node):
+        FedJsonConfigurator.process_config_element(self, config_ctx, node)
+
+        element = node.element
+        path = node.path()
+
+        # executors
+        if re.search(r"^executors\.#[0-9]+$", path):
+            self.current_exe = _ExecutorDef()
+            node.props['data'] = self.current_exe
+            node.exit_cb = self._process_executor_def
+            return
+
+        if re.search(r"^executors\.#[0-9]+\.tasks$", path):
+            self.current_exe.tasks = element
+            return
+
+        if re.search(r"^executors\.#[0-9]+\.executor$", path):
+            self.current_exe.executor = self.build_component(element)
+            return
+
+    def build_component(self, config_dict):
+        t = super().build_component(config_dict)
+        if isinstance(t, FLComponent):
+            if type(t).__name__ not in [type(h).__name__ for h in self.handlers]:
+                self.handlers.append(t)
+        return t
+
+    def _process_executor_def(self, node: Node):
+        e = node.props['data']
+        assert isinstance(e, _ExecutorDef)
+        self.validate_tasks(e.tasks)
+
+        if not isinstance(e.executor, Executor):
+            raise ConfigError('"executor" must be an Executor object but got {}'.format(type(e.executor)))
+
+        self.executors.append(e)
+
+    def finalize_config(self, config_ctx: ConfigContext):
+        FedJsonConfigurator.finalize_config(self, config_ctx)
+
+        if len(self.executors) <= 0:
+            raise ConfigError('executors are not specified')
+
+        task_table = {}
+        for e in self.executors:
+            for t in e.tasks:
+                if t in task_table:
+                    raise ConfigError('Multiple executors defined for task "{}"'.format(t))
+                task_table[t] = e.executor
+
+        self.runner_config = ClientRunnerConfig(
+            task_table=task_table,
+            task_data_filters=self.data_filter_table,
+            task_result_filters=self.result_filter_table,
+            components=self.components,
+            handlers=self.handlers
+        )
diff --git a/nvflare/private/fed/client/client_req_processors.py b/nvflare/private/fed/client/client_req_processors.py
new file mode 100644
index 0000000000..2024bba59d
--- /dev/null
+++ b/nvflare/private/fed/client/client_req_processors.py
@@ -0,0 +1,59 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .info_coll_cmd import ClientInfoProcessor
+from .process_aux_cmd import AuxRequestProcessor
+from .shell_cmd import ShellCommandProcessor
+from .sys_cmd import SysInfoProcessor
+from .training_cmds import (
+    AbortAppProcessor,
+    ClientStatusProcessor,
+    DeleteRunNumberProcessor,
+    DeployProcessor,
+    RestartClientProcessor,
+    ShutdownClientProcessor,
+    # StartClientMGpuProcessor,
+    StartAppProcessor,
+    AbortTaskProcessor,
+    SetRunNumberProcessor,
+)
+from .validation_cmd import ValidateRequestProcessor
+
+
+class ClientRequestProcessors:
+    request_processors = [
+        StartAppProcessor(),
+        ClientStatusProcessor(),
+        AbortAppProcessor(),
+        ShutdownClientProcessor(),
+        DeployProcessor(),
+        ValidateRequestProcessor(),
+        ShellCommandProcessor(),
+        DeleteRunNumberProcessor(),
+        SysInfoProcessor(),
+        RestartClientProcessor(),
+        # StartClientMGpuProcessor(),
+        ClientInfoProcessor(),
+        AbortTaskProcessor(),
+        SetRunNumberProcessor(),
+        AuxRequestProcessor()
+    ]
+
+    @staticmethod
+    def register_cmd_module(request_processor):
+        from .admin import RequestProcessor
+
+        assert isinstance(request_processor, RequestProcessor)
+
+        ClientRequestProcessors.request_processors.append(request_processor)
diff --git a/nvflare/private/fed/client/client_run_manager.py b/nvflare/private/fed/client/client_run_manager.py
new file mode 100644
index 0000000000..41d7710525
--- /dev/null
+++ b/nvflare/private/fed/client/client_run_manager.py
@@ -0,0 +1,147 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List, Optional
+
+from nvflare.apis.client_engine_spec import TaskAssignment
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext, FLContextManager
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.apis.workspace import Workspace
+from nvflare.private.event import fire_event
+from nvflare.widgets.fed_event import ClientFedEventRunner
+from nvflare.widgets.info_collector import InfoCollector
+from nvflare.widgets.widget import Widget, WidgetID
+from .client_aux_runner import ClientAuxRunner
+from .client_engine_executor_spec import ClientEngineExecutorSpec
+from .client_json_config import ClientJsonConfigurator
+from .client_runner import ClientRunner
+from .fed_client import FederatedClient
+
+
+class ClientRunInfo(object):
+    def __init__(self, run_number):
+        self.run_number = run_number
+        self.current_task_name = ""
+        self.start_time = None
+        # self.status = MachineStatus.STOPPED
+
+
+class ClientRunManager(ClientEngineExecutorSpec):
+    """
+        ClientRunManager provides the ClientEngine APIs implementation running in the child process.
+    """
+    def __init__(
+        self,
+        client_name: str,
+        run_num: int,
+        workspace: Workspace,
+        client: FederatedClient,
+        components: Dict[str, FLComponent],
+        handlers: Optional[List[FLComponent]] = None,
+        conf: ClientJsonConfigurator = None
+    ) -> None:
+        super().__init__()
+
+        self.client = client
+        self.handlers = handlers
+        self.workspace = workspace
+        self.components = components
+        self.aux_runner = ClientAuxRunner()
+        self.add_handler(self.aux_runner)
+        self.conf = conf
+
+        self.fl_ctx_mgr = FLContextManager(
+            engine=self, identity_name=client_name, run_num=run_num, public_stickers={}, private_stickers={}
+        )
+
+        self.run_info = ClientRunInfo(run_number=run_num)
+
+        self.widgets = {
+            WidgetID.INFO_COLLECTOR: InfoCollector(),
+            WidgetID.FED_EVENT_RUNNER: ClientFedEventRunner()
+        }
+        for _, widget in self.widgets.items():
+            self.handlers.append(widget)
+
+    def get_task_assignment(self, fl_ctx: FLContext) -> TaskAssignment:
+        pull_success, task_name, remote_tasks = self.client.fetch_task(fl_ctx)
+        task = None
+        if pull_success:
+            shareable = self.client.extract_shareable(remote_tasks, fl_ctx)
+            # task_id = fl_ctx.get_peer_context().get_cookie(FLContextKey.TASK_ID)
+            task_id = shareable.get_header(key=FLContextKey.TASK_ID)
+            task = TaskAssignment(name=task_name, task_id=task_id, data=shareable)
+        return task
+
+    def new_context(self) -> FLContext:
+        return self.fl_ctx_mgr.new_context()
+
+    def send_task_result(self, result: Shareable, fl_ctx: FLContext) -> bool:
+        try:
+            self.client.push_results(result, fl_ctx)  # push task execution results
+            return True
+        except BaseException:
+            return False
+
+    def get_workspace(self) -> Workspace:
+        return self.workspace
+
+    def get_run_info(self) -> ClientRunInfo:
+        return self.run_info
+
+    def show_errors(self) -> ClientRunInfo:
+        return self.run_info
+
+    def reset_errors(self) -> ClientRunInfo:
+        return self.run_info
+
+    def dispatch(self, topic: str, request: Shareable, fl_ctx: FLContext) -> Shareable:
+        return self.aux_runner.dispatch(topic=topic, request=request, fl_ctx=fl_ctx)
+
+    def get_component(self, component_id: str) -> object:
+        return self.components.get(component_id)
+
+    def get_widget(self, widget_id: str) -> Widget:
+        return self.widgets.get(widget_id)
+
+    def fire_event(self, event_type: str, fl_ctx: FLContext):
+        fire_event(event=event_type, handlers=self.handlers, ctx=fl_ctx)
+
+    def add_handler(self, handler: FLComponent):
+        self.handlers.append(handler)
+
+    def build_component(self, config_dict):
+        if not self.conf:
+            raise RuntimeError("No configurator set up.")
+        return self.conf.build_component(config_dict)
+
+    def aux_send(self, topic: str, request: Shareable, timeout: float, fl_ctx: FLContext) -> Shareable:
+        reply = self.client.aux_send(topic, request, timeout, fl_ctx)
+        if reply:
+            return self.client.extract_shareable(reply, fl_ctx)
+        else:
+            return make_reply(ReturnCode.COMMUNICATION_ERROR)
+
+    def send_aux_request(self, topic: str, request: Shareable, timeout: float, fl_ctx: FLContext) -> Shareable:
+        return self.aux_runner.send_aux_request(topic, request, timeout, fl_ctx)
+
+    def register_aux_message_handler(self, topic: str, message_handle_func):
+        self.aux_runner.register_aux_message_handler(topic, message_handle_func)
+
+    def abort_app(self, run_number: int, fl_ctx: FLContext):
+        runner = fl_ctx.get_prop(key=FLContextKey.RUNNER, default=None)
+        if isinstance(runner, ClientRunner):
+            runner.abort()
diff --git a/nvflare/private/fed/client/client_runner.py b/nvflare/private/fed/client/client_runner.py
new file mode 100644
index 0000000000..1b8b582b4e
--- /dev/null
+++ b/nvflare/private/fed/client/client_runner.py
@@ -0,0 +1,420 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+
+from nvflare.apis.client_engine_spec import ClientEngineSpec, TaskAssignment
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode, ReservedKey, ReservedTopic
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.apis.signal import Signal
+from nvflare.private.defs import SpecialTaskName
+from nvflare.private.defs import TaskConstant
+from nvflare.widgets.info_collector import GroupInfoCollector, InfoCollector
+
+
+class ClientRunnerConfig(object):
+    def __init__(
+        self,
+        task_table: dict,  # task_name => Executor
+        task_data_filters: dict,  # task_name => list of filters
+        task_result_filters: dict,  # task_name => list of filters
+        handlers=None,  # list of event handlers
+        components=None,  # dict of extra python objects: id => object
+    ):
+        self.task_table = task_table
+        self.task_data_filters = task_data_filters
+        self.task_result_filters = task_result_filters
+        self.handlers = handlers
+        self.components = components
+
+
+class ClientRunner(FLComponent):
+    def __init__(
+        self,
+        config: ClientRunnerConfig,
+        run_num: int,
+        engine: ClientEngineSpec,
+        task_fetch_interval: int = 5,  # fetch task every 5 secs
+    ):
+        FLComponent.__init__(self)
+        self.task_table = config.task_table
+        self.task_data_filters = config.task_data_filters
+        self.task_result_filters = config.task_result_filters
+
+        self.run_num = run_num
+        self.engine = engine
+        self.task_fetch_interval = task_fetch_interval
+        self.run_abort_signal = Signal()
+        self.task_abort_signal = None
+        self.current_executor = None
+        self.current_task = None
+        self.asked_to_stop = False
+        self.task_lock = threading.Lock()
+
+        engine.register_aux_message_handler(
+            topic=ReservedTopic.END_RUN,
+            message_handle_func=self._handle_end_run
+        )
+        engine.register_aux_message_handler(
+            topic=ReservedTopic.ABORT_ASK,
+            message_handle_func=self._handle_abort_task
+        )
+
+    def _process_task(self, task: TaskAssignment, fl_ctx: FLContext) -> Shareable:
+        engine = fl_ctx.get_engine()
+        assert isinstance(engine, ClientEngineSpec)
+
+        if not isinstance(task.data, Shareable):
+            self.log_error(
+                fl_ctx,
+                'got invalid task data in assignment: expect Shareable, but got {}'.format(
+                    type(task.data))
+            )
+            return make_reply(ReturnCode.BAD_TASK_DATA)
+
+        fl_ctx.set_prop(FLContextKey.TASK_DATA, value=task.data, private=True, sticky=False)
+        fl_ctx.set_prop(FLContextKey.TASK_NAME, value=task.name, private=True, sticky=False)
+        fl_ctx.set_prop(FLContextKey.TASK_ID, value=task.task_id, private=True, sticky=False)
+
+        peer_ctx = fl_ctx.get_peer_context()
+        if not peer_ctx:
+            self.log_error(fl_ctx, 'missing peer context in Server task assignment')
+            return make_reply(ReturnCode.MISSING_PEER_CONTEXT)
+
+        if not isinstance(peer_ctx, FLContext):
+            self.log_error(
+                fl_ctx,
+                "bad peer context in Server task assignment: expects FLContext but got {}".format(type(peer_ctx)),
+            )
+            return make_reply(ReturnCode.BAD_PEER_CONTEXT)
+
+        task.data.set_peer_props(peer_ctx.get_all_public_props())
+        peer_run_num = peer_ctx.get_run_number()
+        if peer_run_num != self.run_num:
+            self.log_error(fl_ctx, "bad task assignment: not for the same run_number")
+            return make_reply(ReturnCode.RUN_MISMATCH)
+
+        executor = self.task_table.get(task.name)
+        if not executor:
+            self.log_error(
+                fl_ctx,
+                "bad task assignment: no executor available for task {}".format(task.name))
+            return make_reply(ReturnCode.TASK_UNKNOWN)
+
+        self.log_debug(fl_ctx, "firing event EventType.BEFORE_TASK_DATA_FILTER")
+        self.fire_event(EventType.BEFORE_TASK_DATA_FILTER, fl_ctx)
+        filter_list = self.task_data_filters.get(task.name)
+        if filter_list:
+            task_data = task.data
+            for f in filter_list:
+                try:
+                    task_data = f.process(task_data, fl_ctx)
+                except BaseException:
+                    self.log_exception(
+                        fl_ctx,
+                        "processing error in Task Data Filter {}".format(type(f))
+                    )
+                    return make_reply(ReturnCode.TASK_DATA_FILTER_ERROR)
+
+            if not isinstance(task_data, Shareable):
+                self.log_error(
+                    fl_ctx,
+                    'task data was converted to wrong type: expect Shareable but got {}'.format(
+                        type(task_data)
+                    ))
+                return make_reply(ReturnCode.TASK_DATA_FILTER_ERROR)
+
+            task.data = task_data
+
+        self.log_debug(fl_ctx, "firing event EventType.AFTER_TASK_DATA_FILTER")
+        fl_ctx.set_prop(FLContextKey.TASK_DATA, value=task.data, private=True, sticky=False)
+        self.fire_event(EventType.AFTER_TASK_DATA_FILTER, fl_ctx)
+
+        self.log_debug(fl_ctx, "firing event EventType.BEFORE_TASK_EXECUTION")
+        fl_ctx.set_prop(FLContextKey.TASK_DATA, value=task.data, private=True, sticky=False)
+        self.fire_event(EventType.BEFORE_TASK_EXECUTION, fl_ctx)
+        try:
+            self.log_info(
+                fl_ctx,
+                "invoking task executor {}".format(type(executor)))
+
+            with self.task_lock:
+                self.task_abort_signal = Signal()
+                self.current_executor = executor
+                self.current_task = task
+
+            try:
+                reply = executor.execute(task.name, task.data, fl_ctx, self.task_abort_signal)
+            finally:
+                with self.task_lock:
+                    self.task_abort_signal = None
+                    self.current_task = None
+                    self.current_executor = None
+
+            if not isinstance(reply, Shareable):
+                self.log_error(
+                    fl_ctx,
+                    'bad result generated by executor {}: must be Shareable but got {}'.format(
+                        type(executor), type(reply)
+                    )
+                )
+                return make_reply(ReturnCode.EXECUTION_RESULT_ERROR)
+        except RuntimeError as e:
+            self.log_exception(
+                fl_ctx,
+                f"Critical RuntimeError happened with Exception {e}: Aborting the RUN!")
+            self.asked_to_stop = True
+            return make_reply(ReturnCode.EXECUTION_RESULT_ERROR)
+        except BaseException:
+            self.log_exception(
+                fl_ctx,
+                "processing error in task executor {}".format(type(executor)))
+            return make_reply(ReturnCode.EXECUTION_EXCEPTION)
+
+        fl_ctx.set_prop(FLContextKey.TASK_RESULT, value=reply, private=True, sticky=False)
+
+        self.log_debug(fl_ctx, "firing event EventType.AFTER_TASK_EXECUTION")
+        self.fire_event(EventType.AFTER_TASK_EXECUTION, fl_ctx)
+
+        self.log_debug(fl_ctx, "firing event EventType.BEFORE_TASK_RESULT_FILTER")
+        self.fire_event(EventType.BEFORE_TASK_RESULT_FILTER, fl_ctx)
+        filter_list = self.task_result_filters.get(task.name)
+        if filter_list:
+            for f in filter_list:
+                try:
+                    reply = f.process(reply, fl_ctx)
+                except BaseException:
+                    self.log_exception(
+                        fl_ctx,
+                        "processing error in Task Result Filter {}".format(type(f))
+                    )
+                    return make_reply(ReturnCode.TASK_RESULT_FILTER_ERROR)
+
+            if not isinstance(reply, Shareable):
+                self.log_error(
+                    fl_ctx,
+                    'task result was converted to wrong type: expect Shareable but got {}'.format(
+                        type(reply)
+                    ))
+
+                return make_reply(ReturnCode.TASK_RESULT_FILTER_ERROR)
+
+        fl_ctx.set_prop(FLContextKey.TASK_RESULT, value=reply, private=True, sticky=False)
+
+        self.log_debug(fl_ctx, "firing event EventType.AFTER_TASK_RESULT_FILTER")
+        self.fire_event(EventType.AFTER_TASK_RESULT_FILTER, fl_ctx)
+        self.log_info(fl_ctx, "finished processing task")
+
+        if not isinstance(reply, Shareable):
+            self.log_error(
+                fl_ctx,
+                'task processing error: expects result to be Shareable, but got {}'.format(type(reply))
+            )
+            return make_reply(ReturnCode.EXECUTION_RESULT_ERROR)
+
+        return reply
+
+    def _try_run(self):
+        task_fetch_interval = self.task_fetch_interval
+        while not self.asked_to_stop:
+            with self.engine.new_context() as fl_ctx:
+                if self.run_abort_signal.triggered:
+                    self.log_info(fl_ctx, "run abort signal received")
+                    break
+
+                time.sleep(task_fetch_interval)
+
+                if self.run_abort_signal.triggered:
+                    self.log_info(fl_ctx, "run abort signal received")
+                    break
+
+                # reset to default fetch interval
+                task_fetch_interval = self.task_fetch_interval
+                self.log_info(fl_ctx, "fetching task from server ...")
+                task = self.engine.get_task_assignment(fl_ctx)
+                if not task:
+                    self.log_info(fl_ctx, "no task received - will try in {} secs".format(task_fetch_interval))
+                    continue
+
+                if task.name == SpecialTaskName.END_RUN:
+                    self.log_info(fl_ctx, "server asked to end the run")
+                    break
+
+                if task.name == SpecialTaskName.TRY_AGAIN:
+                    task_data = task.data
+                    if task_data and isinstance(task_data, Shareable):
+                        task_fetch_interval = task_data.get(TaskConstant.WAIT_TIME, self.task_fetch_interval)
+                    self.log_info(fl_ctx, "server asked to try again - will try in {} secs".format(task_fetch_interval))
+                    continue
+
+                self.log_info(fl_ctx, "got task assignment: name={}, id={}".format(task.name, task.task_id))
+
+                # create a new task abort signal
+                task_reply = self._process_task(task, fl_ctx)
+
+                assert isinstance(task_reply, Shareable)
+                self.log_debug(fl_ctx, 'firing event EventType.BEFORE_SEND_TASK_RESULT')
+                self.fire_event(EventType.BEFORE_SEND_TASK_RESULT, fl_ctx)
+
+                # set the cookie in the reply!
+                task_data = task.data
+                assert isinstance(task_data, Shareable)
+
+                cookie_jar = task_data.get_cookie_jar()
+                if cookie_jar:
+                    task_reply.set_cookie_jar(cookie_jar)
+
+                reply_sent = self.engine.send_task_result(task_reply, fl_ctx)
+                if reply_sent:
+                    self.log_info(
+                        fl_ctx, "result sent to server for task: name={}, id={}".format(task.name, task.task_id)
+                    )
+                else:
+                    self.log_error(
+                        fl_ctx,
+                        "failed to send result to server for task: name={}, id={}".format(task.name, task.task_id),
+                    )
+                self.log_debug(fl_ctx, "firing event EventType.AFTER_SEND_TASK_RESULT")
+                self.fire_event(EventType.AFTER_SEND_TASK_RESULT, fl_ctx)
+
+    def run(self, app_root, args):
+        with self.engine.new_context() as fl_ctx:
+            fl_ctx.set_prop(FLContextKey.APP_ROOT, app_root, sticky=True)
+            fl_ctx.set_prop(FLContextKey.ARGS, args, sticky=True)
+            fl_ctx.set_prop(ReservedKey.RUN_ABORT_SIGNAL, self.run_abort_signal,
+                            private=True, sticky=True)
+            self.log_debug(fl_ctx, "firing event EventType.START_RUN")
+            self.fire_event(EventType.START_RUN, fl_ctx)
+            self.log_info(fl_ctx, "client runner started")
+
+        try:
+            self._try_run()
+        except BaseException:
+            with self.engine.new_context() as fl_ctx:
+                self.log_exception(fl_ctx, "processing error in RUN execution")
+        finally:
+            # in case any task is still running, abort it
+            self._abort_current_task()
+
+            with self.engine.new_context() as fl_ctx:
+                self.fire_event(EventType.END_RUN, fl_ctx)
+                self.log_info(fl_ctx, "current RUN ended")
+
+    def _abort_current_task(self):
+        with self.task_lock:
+            task_abort_signal = self.task_abort_signal
+            if task_abort_signal:
+                # set task_abort_signal to None to prevent triggering again
+                self.task_abort_signal = None
+                task_name = ''
+                task_id = ''
+                task = self.current_task
+                if task:
+                    task_name = task.name
+                    task_id = task.task_id
+
+                with self.engine.new_context() as fl_ctx:
+                    fl_ctx.set_prop(FLContextKey.TASK_NAME, value=task_name, private=True, sticky=False)
+                    fl_ctx.set_prop(FLContextKey.TASK_ID, value=task_id, private=True, sticky=False)
+
+                    task_abort_signal.trigger(True)
+                    self.log_info(
+                        fl_ctx,
+                        "triggered task_abort_signal to stop task '{}'".format(task_name))
+
+                    self.fire_event(EventType.ABORT_TASK, fl_ctx)
+                    self.log_info(
+                        fl_ctx,
+                        "fired ABORT_TASK event to abort current task {}".format(task_name))
+
+    def abort_task(self, task_names=None):
+        has_task_to_abort = False
+        with self.engine.new_context() as fl_ctx:
+            with self.task_lock:
+                if self.current_task:
+                    name = self.current_task.name
+                    if not task_names or name in task_names:
+                        has_task_to_abort = True
+                    else:
+                        self.log_info(
+                            fl_ctx,
+                            "Ignored abort_task request since current task '{}' is not target".format(name))
+                else:
+                    self.log_info(
+                        fl_ctx,
+                        "Ignored abort_task request since there is no current task")
+
+        if has_task_to_abort:
+            self._abort_current_task()
+
+    def abort(self):
+        """
+        Abort the current run
+
+        Returns:
+
+        """
+        self._abort_current_task()
+        with self.engine.new_context() as fl_ctx:
+            self.fire_event(EventType.END_RUN, fl_ctx)
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == InfoCollector.EVENT_TYPE_GET_STATS:
+            collector = fl_ctx.get_prop(InfoCollector.CTX_KEY_STATS_COLLECTOR)
+            if collector:
+                assert isinstance(collector, GroupInfoCollector)
+                if self.current_task:
+                    current_task_name = self.current_task.name
+                else:
+                    current_task_name = "None"
+                collector.set_info(
+                    group_name="ClientRunner",
+                    info={"run_number": self.run_num,
+                          "current_task_name": current_task_name,
+                          "status": "started"
+                          },
+                )
+        elif event_type == EventType.FATAL_TASK_ERROR:
+            reason = fl_ctx.get_prop(key=FLContextKey.EVENT_DATA, default='')
+            self.log_error(fl_ctx,
+                           "Aborting current task due to FATAL_TASK_ERROR received: {}".format(reason))
+            self._abort_current_task()
+        elif event_type == EventType.FATAL_SYSTEM_ERROR:
+            reason = fl_ctx.get_prop(key=FLContextKey.EVENT_DATA, default='')
+            self.log_error(fl_ctx,
+                           "Aborting current RUN due to FATAL_SYSTEM_ERROR received: {}".format(reason))
+            self.abort()
+
+    def _handle_end_run(self,
+                        topic: str,
+                        request: Shareable,
+                        fl_ctx: FLContext) -> Shareable:
+        self.log_info(fl_ctx,
+                      "received aux request from Server to end current RUN")
+        self.abort()
+        return make_reply(ReturnCode.OK)
+
+    def _handle_abort_task(self,
+                           topic: str,
+                           request: Shareable,
+                           fl_ctx: FLContext) -> Shareable:
+        self.log_info(fl_ctx,
+                      "received aux request from Server to abort current task")
+        task_names = request.get('task_names', None)
+        self.abort_task(task_names)
+        return make_reply(ReturnCode.OK)
diff --git a/nvflare/private/fed/client/client_status.py b/nvflare/private/fed/client/client_status.py
new file mode 100644
index 0000000000..6c6dda6a51
--- /dev/null
+++ b/nvflare/private/fed/client/client_status.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class ClientStatus(object):
+    NOT_STARTED = 0
+    STARTING = 1
+    STARTED = 2
+    STOPPED = 3
+    EXCEPTION = 4
+
+    status_messages = {
+        NOT_STARTED: "not started",
+        STARTING: "starting",
+        STARTED: "started",
+        STOPPED: "stopped",
+        EXCEPTION: "exception",
+    }
+
+
+def get_status_message(status):
+    return ClientStatus.status_messages.get(status)
diff --git a/nvflare/private/fed/client/command_agent.py b/nvflare/private/fed/client/command_agent.py
new file mode 100644
index 0000000000..6054aa82b0
--- /dev/null
+++ b/nvflare/private/fed/client/command_agent.py
@@ -0,0 +1,78 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+from multiprocessing.connection import Listener
+
+from nvflare.apis.fl_context import FLContext
+from .admin_commands import AdminCommands
+
+
+class CommandAgent(object):
+    def __init__(self, federated_client, listen_port, client_runner) -> None:
+        self.federated_client = federated_client
+        self.listen_port = int(listen_port)
+        self.client_runner = client_runner
+        self.thread = None
+        self.asked_to_stop = False
+
+        self.commands = AdminCommands.commands
+
+    def start(self, fl_ctx: FLContext):
+        # self.thread = threading.Thread(target=listen_command, args=[federated_client, int(listen_port), client_runner])
+        self.thread = threading.Thread(target=listen_command, args=[self, fl_ctx])
+        self.thread.start()
+
+        pass
+
+    def listen_command(self, fl_ctx):
+        try:
+            address = ("localhost", self.listen_port)  # family is deduced to be 'AF_INET'
+            listener = Listener(address, authkey="client process secret password".encode())
+            conn = listener.accept()
+            print(f"Created the listener on port: {self.listen_port}")
+
+            try:
+                while not self.asked_to_stop:
+                    if conn.poll(1.0):
+                        msg = conn.recv()
+                        command_name = msg.get("command")
+                        data = msg.get("data")
+                        command = AdminCommands.get_command(command_name)
+                        if command:
+                            engine = fl_ctx.get_engine()
+                            with engine.new_context() as new_fl_ctx:
+                                reply = command.process(data=data, fl_ctx=new_fl_ctx)
+                                if reply:
+                                    conn.send(reply)
+            except Exception as e:
+                # traceback.print_exc()
+                print(f"Process communication exception: {self.listen_port}.")
+            finally:
+                conn.close()
+
+            listener.close()
+        except Exception as e:
+            print(f"Could not create the listener for this process on port: {self.listen_port}.")
+            pass
+
+    def shutdown(self):
+        self.asked_to_stop = True
+
+        if self.thread and self.thread.is_alive():
+            self.thread.join()
+
+
+def listen_command(agent: CommandAgent, fl_ctx: FLContext):
+    agent.listen_command(fl_ctx)
diff --git a/nvflare/private/fed/client/communicator.py b/nvflare/private/fed/client/communicator.py
new file mode 100644
index 0000000000..b28c1222ec
--- /dev/null
+++ b/nvflare/private/fed/client/communicator.py
@@ -0,0 +1,403 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import math
+import socket
+import time
+from typing import List, Optional
+
+import grpc
+from google.protobuf.struct_pb2 import Struct
+
+import nvflare.private.fed.protos.federated_pb2 as fed_msg
+import nvflare.private.fed.protos.federated_pb2_grpc as fed_service
+from nvflare.apis.filter import Filter
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.fl_exception import FLCommunicationError
+from nvflare.private.fed.utils.fed_utils import shareable_to_modeldata, make_context_data, make_shareeable_data
+
+
+class Communicator:
+    def __init__(
+        self,
+        ssl_args=None,
+        secure_train=False,
+        retry_timeout=30,
+        client_state_processors: Optional[List[Filter]] = None,
+        compression=None,
+    ):
+        self.ssl_args = ssl_args
+        self.secure_train = secure_train
+
+        self.verbose = False
+        self.should_stop = False
+        self.heartbeat_done = False
+        self.retry = int(math.ceil(float(retry_timeout) / 5))
+        self.client_state_processors = client_state_processors
+        self.compression = compression
+
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def set_up_channel(self, channel_dict, token=None):
+        """
+        Connect client to the server.
+
+        :param channel_dict: grpc channel parameters
+        :param token: client token
+        :return: an initialised grpc channel
+        """
+        if self.secure_train:
+            with open(self.ssl_args["ssl_root_cert"], "rb") as f:
+                trusted_certs = f.read()
+            with open(self.ssl_args["ssl_private_key"], "rb") as f:
+                private_key = f.read()
+            with open(self.ssl_args["ssl_cert"], "rb") as f:
+                certificate_chain = f.read()
+
+            credentials = grpc.ssl_channel_credentials(
+                certificate_chain=certificate_chain, private_key=private_key, root_certificates=trusted_certs
+            )
+
+            # make sure that all headers are in lowecase,
+            # otherwise grpc throws an exception
+            call_credentials = grpc.metadata_call_credentials(
+                lambda context, callback: callback((("x-custom-token", token),), None)
+            )
+            # use this if you want standard "Authorization" header
+            # call_credentials = grpc.access_token_call_credentials(
+            #     "x-custom-token")
+            composite_credentials = grpc.composite_channel_credentials(credentials, call_credentials)
+            channel = grpc.secure_channel(
+                **channel_dict, credentials=composite_credentials, compression=self.compression
+            )
+
+        else:
+            channel = grpc.insecure_channel(**channel_dict, compression=self.compression)
+        return channel
+
+    def get_client_state(self, project_name, token, fl_ctx: FLContext):
+        """
+        Client's meta data used to authenticate and communicate.
+
+        :return: a ClientState message.
+        """
+        state_message = fed_msg.ClientState(token=token)
+        state_message.meta.project.name = project_name
+        # if self.client_state_processors and app_ctx:
+        #     for t in self.client_state_processors:
+        #         state_message = t.process(state_message, app_ctx)
+        # state_message.meta.run_number = self.run_number
+        state_message.meta.run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+
+        context_data = make_context_data(fl_ctx)
+        state_message.context["fl_context"].CopyFrom(context_data)
+
+        return state_message
+
+    def client_registration(self, client_name, servers, project_name):
+        """
+        Client's meta data used to authenticate and communicate.
+
+        :return: a ClientLogin message.
+        """
+        local_ip = socket.gethostbyname(socket.gethostname())
+        login_message = fed_msg.ClientLogin(client_name=client_name, client_ip=local_ip)
+        # login_message = fed_msg.ClientLogin(
+        #     client_id=None, token=None, client_ip=local_ip)
+        login_message.meta.project.name = project_name
+
+        result, retry = None, self.retry
+        retry = 1500  # retry register for 2 hours (7500s)
+        with self.set_up_channel(servers[project_name]) as channel:
+            stub = fed_service.FederatedTrainingStub(channel)
+            while retry > 0:
+                try:
+                    start_time = time.time()
+                    result = stub.Register(login_message)
+                    token = result.token
+                    self.should_stop = False
+                    break
+                except grpc.RpcError as grpc_error:
+                    self.grpc_error_handler(
+                        servers[project_name],
+                        grpc_error,
+                        "client_registration",
+                        start_time,
+                        retry,
+                        verbose=self.verbose,
+                    )
+                    excep = FLCommunicationError(grpc_error)
+                    if isinstance(grpc_error, grpc.Call):
+                        status_code = grpc_error.code()
+                        if grpc.StatusCode.UNAUTHENTICATED == status_code:
+                            raise excep
+                    retry -= 1
+                    time.sleep(5)
+            if self.should_stop:
+                raise excep
+            if result is None:
+                return None
+
+        return token
+
+    def getTask(self, servers, project_name, token, fl_ctx: FLContext):
+        """
+        Get registered with the remote server via channel,
+        and fetch the server's model parameters.
+
+        :param project_name: server identifier string
+        :return: a CurrentTask message from server
+        """
+        global_model, retry = None, self.retry
+        with self.set_up_channel(servers[project_name]) as channel:
+            stub = fed_service.FederatedTrainingStub(channel)
+            while retry > 0:
+                # get the global model
+                try:
+                    start_time = time.time()
+                    global_model = stub.GetTask(self.get_client_state(project_name, token, fl_ctx))
+                    # Clear the stopping flag
+                    # if the connection to server recovered.
+                    self.should_stop = False
+
+                    end_time = time.time()
+                    self.logger.info(
+                        f"Received from {project_name} server "
+                        f" ({global_model.ByteSize()} Bytes). getTask time: {end_time - start_time} seconds"
+                    )
+
+                    task = fed_msg.CurrentTask()
+                    task.meta.CopyFrom(global_model.meta)
+                    task.meta_data.CopyFrom(global_model.meta_data)
+                    task.data.CopyFrom(global_model.data)
+                    task.task_name = global_model.task_name
+
+                    return task
+                except grpc.RpcError as grpc_error:
+                    self.grpc_error_handler(
+                        servers[project_name], grpc_error, "getTask", start_time, retry, verbose=self.verbose
+                    )
+                    excep = FLCommunicationError(grpc_error)
+                    retry -= 1
+                    time.sleep(5)
+            if self.should_stop:
+                raise excep
+
+        # Failed to get global, return None
+        return None
+
+    def submitUpdate(self, servers, project_name, token, fl_ctx: FLContext, client_name, shareable, execute_task_name):
+        """
+        Submit the task execution result back to the server.
+        Args:
+            servers: FL servers
+            project_name: server project name
+            token: client token
+            fl_ctx: fl_ctx
+            client_name: client name
+            shareable: execution task result shareable
+            execute_task_name: execution task name
+
+        Returns: server message from the server
+
+        """
+        client_state = self.get_client_state(project_name, token, fl_ctx)
+        client_state.client_name = client_name
+        contrib = self._get_communication_data(shareable, client_state, fl_ctx, execute_task_name)
+
+        server_msg, retry = None, self.retry
+        with self.set_up_channel(servers[project_name]) as channel:
+            stub = fed_service.FederatedTrainingStub(channel)
+            while retry > 0:
+                try:
+                    start_time = time.time()
+                    self.logger.info(f"Send submitUpdate to {project_name} server")
+                    server_msg = stub.SubmitUpdate(contrib)
+                    # Clear the stopping flag
+                    # if the connection to server recovered.
+                    self.should_stop = False
+
+                    end_time = time.time()
+                    self.logger.info(
+                        f"Received comments: {server_msg.meta.project.name} {server_msg.comment}."
+                        f" SubmitUpdate time: {end_time - start_time} seconds"
+                    )
+                    break
+                except grpc.RpcError as grpc_error:
+                    if isinstance(grpc_error, grpc.Call):
+                        if grpc_error.details().startswith("Contrib"):
+                            self.logger.info(f"submitUpdate failed: {grpc_error.details()}")
+                            break  # outdated contribution, no need to retry
+                    self.grpc_error_handler(
+                        servers[project_name], grpc_error, "submitUpdate", start_time, retry, verbose=self.verbose
+                    )
+                    retry -= 1
+                    time.sleep(5)
+        return server_msg
+
+    def auxCommunicate(self, servers, project_name, token, fl_ctx: FLContext, client_name, shareable, topic, timeout):
+        """
+        send the aux communication message to the server
+        Args:
+            servers: FL servers
+            project_name: server project name
+            token: client token
+            fl_ctx: fl_ctx
+            client_name: client name
+            shareable: aux message shareable
+            topic: aux message topic
+            timeout: aux communication timeout
+
+        Returns: servre response message
+
+        """
+        client_state = self.get_client_state(project_name, token, fl_ctx)
+        client_state.client_name = client_name
+
+        aux_message = fed_msg.AuxMessage()
+        # set client auth. data
+        aux_message.client.CopyFrom(client_state)
+
+        # shareable.set_header("Topic", topic)
+        aux_message.data["data"].CopyFrom(make_shareeable_data(shareable))
+        aux_message.data["fl_context"].CopyFrom(make_context_data(fl_ctx))
+
+        server_msg, retry = None, self.retry
+        with self.set_up_channel(servers[project_name]) as channel:
+            stub = fed_service.FederatedTrainingStub(channel)
+            while retry > 0:
+                try:
+                    start_time = time.time()
+                    self.logger.info(f"Send AuxMessage to {project_name} server")
+                    server_msg = stub.AuxCommunicate(aux_message, timeout=timeout)
+                    # Clear the stopping flag
+                    # if the connection to server recovered.
+                    self.should_stop = False
+
+                    break
+                except grpc.RpcError as grpc_error:
+                    self.grpc_error_handler(
+                        servers[project_name], grpc_error, "AuxCommunicate", start_time, retry, verbose=self.verbose
+                    )
+                    retry -= 1
+                    time.sleep(5)
+        return server_msg
+
+    def quit_remote(self, servers, task_name, token, fl_ctx: FLContext):
+        """
+        Sending the last message to the server before leaving.
+
+        :param task_name: server task identifier
+        :return: server's reply to the last message
+        """
+        server_message, retry = None, self.retry
+        with self.set_up_channel(servers[task_name]) as channel:
+            stub = fed_service.FederatedTrainingStub(channel)
+            while retry > 0:
+                try:
+                    start_time = time.time()
+                    self.logger.info(f"Quitting server: {task_name}")
+                    server_message = stub.Quit(self.get_client_state(task_name, token, fl_ctx))
+                    # Clear the stopping flag
+                    # if the connection to server recovered.
+                    self.should_stop = False
+
+                    end_time = time.time()
+                    self.logger.info(
+                        f"Received comment from server: {server_message.comment}. Quit time: {end_time - start_time} seconds"
+                    )
+                    break
+                except grpc.RpcError as grpc_error:
+                    self.grpc_error_handler(servers[task_name], grpc_error, "quit_remote", start_time, retry)
+                    retry -= 1
+                    time.sleep(3)
+        return server_message
+
+    def send_heartbeat(self, servers, task_name, token, client_name):
+        message = fed_msg.Token()
+        message.token = token
+        message.client_name = client_name
+
+        with self.set_up_channel(servers[task_name]) as channel:
+            while not self.heartbeat_done:
+                stub = fed_service.FederatedTrainingStub(channel)
+                # retry the heartbeat call for 10 minutes
+                retry = 120
+                while retry > 0:
+                    try:
+                        self.logger.debug(f"Send {task_name} heartbeat {token}")
+                        stub.Heartbeat(message)
+                        break
+                    except grpc.RpcError as grpc_error:
+                        self.logger.debug(grpc_error)
+                        excep = FLCommunicationError(grpc_error)
+                        retry -= 1
+                        time.sleep(5)
+                        # pass
+                # if retry <= 0:
+                #     raise excep
+
+                time.sleep(60)
+
+    def _get_communication_data(self, shareable, client_state, fl_ctx: FLContext, execute_task_name):
+        contrib = fed_msg.Contribution()
+        # set client auth. data
+        contrib.client.CopyFrom(client_state)
+        contrib.task_name = execute_task_name
+
+        current_model = shareable_to_modeldata(shareable, fl_ctx)
+        contrib.data.CopyFrom(current_model)
+
+        s = Struct()
+        contrib.meta_data.CopyFrom(s)
+
+        return contrib
+
+    def grpc_error_handler(self, service, grpc_error, action, start_time, retry, verbose=False):
+        """
+        Handling grpc exceptions
+        :param action:
+        :param start_time:
+        :param service:
+        """
+        status_code = None
+        if isinstance(grpc_error, grpc.Call):
+            status_code = grpc_error.code()
+
+        if grpc.StatusCode.RESOURCE_EXHAUSTED == status_code:
+            if grpc_error.details().startswith("No token"):
+                self.logger.info("No token for this client in current round. " "Waiting for server new round. ")
+                self.should_stop = False
+                return
+
+        self.logger.error(
+            f"Action: {action} grpc communication error. retry: {retry}, First start till now: {time.time() - start_time} seconds."
+        )
+        if grpc.StatusCode.UNAVAILABLE == status_code:
+            self.logger.error(
+                f"Could not connect to server: {service.get('target')}\t"
+                f"Setting flag for stopping training. {grpc_error.details()}"
+            )
+            self.should_stop = True
+
+        if grpc.StatusCode.OUT_OF_RANGE == status_code:
+            self.logger.error(
+                f"Server training has stopped.\t" f"Setting flag for stopping training. {grpc_error.details()}"
+            )
+            self.should_stop = True
+
+        if verbose:
+            self.logger.info(grpc_error)
diff --git a/nvflare/private/fed/client/comp_caller_cmd.py b/nvflare/private/fed/client/comp_caller_cmd.py
new file mode 100644
index 0000000000..d913540cc1
--- /dev/null
+++ b/nvflare/private/fed/client/comp_caller_cmd.py
@@ -0,0 +1,44 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.private.admin_defs import Message
+from nvflare.private.defs import ComponentCallerTopic, RequestHeader
+from nvflare.private.fed.client.admin import RequestProcessor
+from nvflare.private.fed.client.client_engine_internal_spec import ClientEngineInternalSpec
+from nvflare.widgets.comp_caller import ComponentCaller
+from nvflare.widgets.widget import WidgetID
+
+
+class ComponentCallerProcessor(RequestProcessor):
+    def get_topics(self) -> [str]:
+        return [ComponentCallerTopic.CALL_COMPONENT]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+        caller = engine.get_widget(WidgetID.COMPONENT_CALLER)
+        assert isinstance(caller, ComponentCaller)
+        run_info = engine.get_current_run_info()
+        if not run_info or run_info.run_number < 0:
+            result = {"error": "app not running"}
+        else:
+            comp_target = req.get_header(RequestHeader.COMPONENT_TARGET)
+            call_name = req.get_header(RequestHeader.CALL_NAME)
+            call_params = req.body
+            result = caller.call_components(target=comp_target, call_name=call_name, params=call_params)
+
+        if not isinstance(result, dict):
+            result = {}
+
+        return Message(topic=req.topic, body=result)
diff --git a/nvflare/private/fed/client/fed_client.py b/nvflare/private/fed/client/fed_client.py
new file mode 100644
index 0000000000..a8321e8daf
--- /dev/null
+++ b/nvflare/private/fed/client/fed_client.py
@@ -0,0 +1,81 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The client of the federated training process"""
+import pickle
+from typing import List, Optional
+
+from nvflare.apis.event_type import EventType
+from nvflare.apis.executor import Executor
+from nvflare.apis.filter import Filter
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.private.event import fire_event
+from .fed_client_base import FederatedClientBase
+from ..utils.numproto import proto_to_bytes
+
+
+class FederatedClient(FederatedClientBase):
+    """
+    Federated client-side implementation.
+    """
+
+    def __init__(
+        self,
+        client_name,
+        client_args,
+        secure_train,
+        server_args=None,
+        retry_timeout=30,
+        client_state_processors: Optional[List[Filter]] = None,
+        handlers: Optional[List[FLComponent]] = None,
+        executors: Optional[List[Executor]] = None,
+        compression=None,
+        enable_byoc=False
+    ):
+        # We call the base implementation directly.
+        super().__init__(
+            client_name=client_name,
+            client_args=client_args,
+            secure_train=secure_train,
+            server_args=server_args,
+            retry_timeout=retry_timeout,
+            client_state_processors=client_state_processors,
+            handlers=handlers,
+            compression=compression,
+        )
+
+        self.executors = executors
+        self.enable_byoc = enable_byoc
+
+    def fetch_task(self, fl_ctx: FLContext):
+        fire_event(EventType.BEFORE_PULL_TASK, self.handlers, fl_ctx)
+
+        pull_success, task_name, remote_tasks = self.pull_task(fl_ctx)
+        fire_event(EventType.AFTER_PULL_TASK, self.handlers, fl_ctx)
+        self.logger.info(f"pull_task completed. Task name:{task_name} Status:{pull_success} ")
+        return pull_success, task_name, remote_tasks
+
+    def extract_shareable(self, responses, fl_ctx: FLContext):
+        shareable = Shareable()
+        peer_context = FLContext()
+        for item in responses:
+            shareable = shareable.from_bytes(proto_to_bytes(item.data.params["data"]))
+            peer_context = pickle.loads(proto_to_bytes(item.data.params["fl_context"]))
+
+        fl_ctx.set_peer_context(peer_context)
+        shareable.set_peer_props(peer_context.get_all_public_props())
+
+        return shareable
diff --git a/nvflare/private/fed/client/fed_client_base.py b/nvflare/private/fed/client/fed_client_base.py
new file mode 100644
index 0000000000..6245cf8d13
--- /dev/null
+++ b/nvflare/private/fed/client/fed_client_base.py
@@ -0,0 +1,278 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The client of the federated training process"""
+
+import logging
+import threading
+from functools import partial
+from multiprocessing.dummy import Pool as ThreadPool
+from typing import List, Optional
+
+from nvflare.apis.filter import Filter
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.fl_exception import FLCommunicationError
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.private.defs import EngineConstant
+from .client_status import ClientStatus
+from .communicator import Communicator
+
+
+class FederatedClientBase:
+    """
+    Federated client-side base implementation.
+    This class provide the tools function which will be used in both FedClient and FedClientLite.
+    """
+
+    def __init__(
+        self,
+        client_name,
+        client_args,
+        secure_train,
+        server_args=None,
+        retry_timeout=30,
+        client_state_processors: Optional[List[Filter]] = None,
+        handlers: Optional[List[FLComponent]] = None,
+        compression=None,
+    ):
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+        self.client_name = client_name
+        self.token = None
+        self.client_args = client_args
+        self.servers = server_args
+
+        self.communicator = Communicator(
+            ssl_args=client_args,
+            secure_train=secure_train,
+            retry_timeout=retry_timeout,
+            client_state_processors=client_state_processors,
+            compression=compression,
+        )
+
+        self.secure_train = secure_train
+        self.handlers = handlers
+
+        self.heartbeat_done = False
+        self.fl_ctx = FLContext()
+        self.platform = None
+        self.abort_signal = Signal()
+
+        self.status = ClientStatus.NOT_STARTED
+
+    def client_register(self, project_name):
+        """
+        Register the client to the FL server and get the FL token.
+        Args:
+            project_name: FL server project name
+
+        Returns:
+
+        """
+        if not self.token:
+            try:
+                self.token = self.communicator.client_registration(self.client_name, self.servers, project_name)
+                if self.token is not None:
+                    self.fl_ctx.set_prop(FLContextKey.CLIENT_NAME, self.client_name, private=False)
+                    self.fl_ctx.set_prop(EngineConstant.FL_TOKEN, self.token, private=False)
+                    self.logger.info(
+                        "Successfully registered client:{} for {}. Got token:{}".format(
+                            self.client_name, project_name, self.token
+                        )
+                    )
+                    # fire_event(EventType.CLIENT_REGISTER, self.handlers, self.fl_ctx)
+
+            except FLCommunicationError as e:
+                self.communicator.heartbeat_done = True
+
+    def fetch_execute_task(self, project_name, fl_ctx: FLContext):
+        """
+        Get registered with the remote server via channel,
+        and fetch the server's model parameters.
+
+        :param project_name: server identifier string
+        :return: a CurrentTask message from server
+        """
+        try:
+            self.logger.info("Starting to fetch execute task.")
+            task = self.communicator.getTask(self.servers, project_name, self.token, fl_ctx)
+
+            return task
+        except FLCommunicationError as e:
+            self.logger.info(e)
+            # self.communicator.heartbeat_done = True
+
+    def push_execute_result(self, project_name, shareable: Shareable, fl_ctx: FLContext):
+        """
+        Read local model and push to self.server[task_name] channel.
+        This function makes and sends a Contribution Message.
+
+        :param project_name: should be one of the keys of `self.server`
+        :param shareable: Shareable object to submit to server
+        """
+        try:
+            self.logger.info("Starting to push execute result.")
+            execute_task_name = fl_ctx.get_prop(FLContextKey.TASK_NAME)
+            message = self.communicator.submitUpdate(
+                self.servers, project_name, self.token, fl_ctx, self.client_name, shareable, execute_task_name
+            )
+
+            return message
+        except FLCommunicationError as e:
+            self.logger.info(e)
+            # self.communicator.heartbeat_done = True
+
+    def send_aux_message(self, project_name, topic: str, shareable: Shareable, timeout: float, fl_ctx: FLContext):
+        """
+        Read local model and push to self.server[task_name] channel.
+        This function makes and sends a Contribution Message.
+
+        :param topic: should be message topic
+        :param timeout: timeout
+        :param project_name: should be one of the keys of `self.server`
+        :param shareable: Shareable object to submit to server
+        """
+        try:
+            self.logger.info("Starting to send aux messagee.")
+            message = self.communicator.auxCommunicate(
+                self.servers, project_name, self.token, fl_ctx, self.client_name, shareable, topic, timeout)
+
+            return message
+        except FLCommunicationError as e:
+            self.logger.info(e)
+            # self.communicator.heartbeat_done = True
+
+    def send_heartbeat(self, project_name):
+        try:
+            if self.token:
+                self.communicator.send_heartbeat(self.servers, project_name, self.token, self.client_name)
+        except FLCommunicationError as e:
+            self.communicator.heartbeat_done = True
+
+    def heartbeat(self):
+        """
+        Sends a heartbeat from the client to the server.
+        """
+        pool = None
+        try:
+            pool = ThreadPool(len(self.servers))
+            return pool.map(self.send_heartbeat, tuple(self.servers))
+        finally:
+            if pool:
+                pool.terminate()
+
+    def pull_task(self, fl_ctx: FLContext):
+        """
+        Fetch remote models and update the local client's session.
+        """
+        pool = None
+        try:
+            pool = ThreadPool(len(self.servers))
+            self.remote_tasks = pool.map(partial(self.fetch_execute_task, fl_ctx=fl_ctx), tuple(self.servers))
+            pull_success, task_name = self.check_progress(self.remote_tasks)
+            # # Update app_ctx's current round info
+            # if self.app_context and self.remote_models[0] is not None:
+            #     self.app_context.global_round = self.remote_models[0].meta.current_round
+            # TODO: if some of the servers failed
+            # return self.model_manager.assign_current_model(self.remote_models)
+            return pull_success, task_name, self.remote_tasks
+        finally:
+            if pool:
+                pool.terminate()
+
+    def push_results(self, shareable: Shareable, fl_ctx: FLContext):
+        """
+        Push the local model to multiple servers.
+        """
+        pool = None
+        try:
+            pool = ThreadPool(len(self.servers))
+            return pool.map(partial(self.push_execute_result, shareable=shareable, fl_ctx=fl_ctx), tuple(self.servers))
+        finally:
+            if pool:
+                pool.terminate()
+
+    def aux_send(self, topic, shareable: Shareable, timeout: float, fl_ctx: FLContext):
+        """
+        Push the local model to multiple servers.
+        """
+        pool = None
+        try:
+            pool = ThreadPool(len(self.servers))
+            messages = pool.map(partial(self.send_aux_message, topic=topic, shareable=shareable,
+                                        timeout=timeout, fl_ctx=fl_ctx), tuple(self.servers))
+            if messages is not None and messages[0] is not None:
+                # Only handle single server communication for now.
+                return messages
+            else:
+                return None
+        finally:
+            if pool:
+                pool.terminate()
+
+    def register(self):
+        """
+        Push the local model to multiple servers.
+        """
+        pool = None
+        try:
+            pool = ThreadPool(len(self.servers))
+            return pool.map(self.client_register, tuple(self.servers))
+        finally:
+            if pool:
+                pool.terminate()
+
+    def run_heartbeat(self):
+        """
+        Periodically runs the heartbeat.
+        """
+        # while not self.heartbeat_done:
+        #     time.sleep(60)
+        #     self.heartbeat()
+
+        self.heartbeat()
+
+        # self.communicator.heartbeat_done = True
+
+    def start_heartbeat(self):
+        heartbeat_thread = threading.Thread(target=self.run_heartbeat)
+        # heartbeat_thread.daemon = True
+        heartbeat_thread.start()
+
+    def quit_remote(self, task_name, fl_ctx: FLContext):
+        """
+        Sending the last message to the server before leaving.
+
+        :param task_name: server task identifier
+        :return: server's reply to the last message
+        """
+        return self.communicator.quit_remote(self.servers, task_name, self.token, fl_ctx)
+
+    def close(self):
+        """
+        Quit the remote federated server, close the local session.
+        """
+        self.logger.info("Shutting down client")
+
+        return 0
+
+    def check_progress(self, remote_tasks):
+        if remote_tasks[0] is not None:
+            self.server_meta = remote_tasks[0].meta
+            return True, remote_tasks[0].task_name
+        else:
+            return False, None
diff --git a/nvflare/private/fed/client/info_coll_cmd.py b/nvflare/private/fed/client/info_coll_cmd.py
new file mode 100644
index 0000000000..f121938d0b
--- /dev/null
+++ b/nvflare/private/fed/client/info_coll_cmd.py
@@ -0,0 +1,82 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from nvflare.private.admin_defs import Message
+from nvflare.private.defs import InfoCollectorTopic
+from nvflare.private.fed.client.admin import RequestProcessor
+from nvflare.private.fed.client.client_engine_internal_spec import ClientEngineInternalSpec
+
+
+class ClientInfoProcessor(RequestProcessor):
+    def get_topics(self) -> [str]:
+        return [
+            InfoCollectorTopic.SHOW_STATS,
+            InfoCollectorTopic.SHOW_ERRORS,
+            InfoCollectorTopic.RESET_ERRORS,
+        ]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+        # run_info = engine.get_current_run_info()
+        # run_num = -1
+        # if not run_info or run_info.run_number < 0:
+        #     result = {
+        #         'error': 'app not running',
+        #     }
+        # else:
+        #     collector = engine.get_widget(WidgetID.INFO_COLLECTOR)
+        #     if not collector:
+        #         result = {
+        #             'error': 'no info collector'
+        #         }
+        #     else:
+        #         assert isinstance(collector, InfoCollector)
+        #         run_num = run_info.run_number
+        #
+        #         if req.topic == InfoCollectorTopic.SHOW_STATS:
+        #             result = collector.get_run_stats()
+        #         elif req.topic == InfoCollectorTopic.SHOW_ERRORS:
+        #             result = collector.get_errors()
+        #         elif req.topic == InfoCollectorTopic.RESET_ERRORS:
+        #             collector.reset_errors()
+        #             result = {
+        #                 'status': 'OK'
+        #             }
+        #         else:
+        #             result = {
+        #                 'error': 'invalid topic {}'.format(req.topic)
+        #             }
+
+        if req.topic == InfoCollectorTopic.SHOW_STATS:
+            result = engine.get_current_run_info()
+        elif req.topic == InfoCollectorTopic.SHOW_ERRORS:
+            result = engine.get_errors()
+        elif req.topic == InfoCollectorTopic.RESET_ERRORS:
+            engine.reset_errors()
+            result = {"status": "OK"}
+        else:
+            result = {"error": "invalid topic {}".format(req.topic)}
+
+        if not isinstance(result, dict):
+            result = {}
+
+        # # add current run number
+        # if run_num >= 0:
+        #     result['run_num'] = run_num
+
+        result = json.dumps(result)
+        return Message(topic="reply_" + req.topic, body=result)
diff --git a/nvflare/private/fed/client/process_aux_cmd.py b/nvflare/private/fed/client/process_aux_cmd.py
new file mode 100644
index 0000000000..a44203cba7
--- /dev/null
+++ b/nvflare/private/fed/client/process_aux_cmd.py
@@ -0,0 +1,39 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pickle
+
+from nvflare.apis.fl_constant import ReservedTopic, ReturnCode
+from nvflare.apis.shareable import make_reply
+from nvflare.private.admin_defs import Message
+
+from .admin import RequestProcessor
+
+
+class AuxRequestProcessor(RequestProcessor):
+    def get_topics(self) -> [str]:
+        return [ReservedTopic.AUX_COMMAND]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+
+        shareable = pickle.loads(req.body)
+
+        result = engine.send_aux_command(shareable)
+        if not result:
+            result = make_reply(ReturnCode.EXECUTION_EXCEPTION)
+
+        result = pickle.dumps(result)
+        message = Message(topic="reply_" + req.topic, body=result)
+        return message
diff --git a/nvflare/private/fed/client/shell_cmd.py b/nvflare/private/fed/client/shell_cmd.py
new file mode 100644
index 0000000000..078f7890d6
--- /dev/null
+++ b/nvflare/private/fed/client/shell_cmd.py
@@ -0,0 +1,29 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+
+from nvflare.private.admin_defs import Message
+from nvflare.private.defs import SysCommandTopic
+from .admin import RequestProcessor
+
+
+class ShellCommandProcessor(RequestProcessor):
+    def get_topics(self) -> [str]:
+        return [SysCommandTopic.SHELL]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        shell_cmd = req.body
+        output = subprocess.getoutput(shell_cmd)
+        return Message(topic="reply_" + req.topic, body=output)
diff --git a/nvflare/private/fed/client/sys_cmd.py b/nvflare/private/fed/client/sys_cmd.py
new file mode 100644
index 0000000000..a407f0d0ff
--- /dev/null
+++ b/nvflare/private/fed/client/sys_cmd.py
@@ -0,0 +1,54 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import psutil
+
+try:
+    import pynvml
+except ImportError:
+    pynvml = None
+
+from nvflare.private.admin_defs import Message
+from nvflare.private.defs import SysCommandTopic
+
+from .admin import RequestProcessor
+
+
+class SysInfoProcessor(RequestProcessor):
+    def get_topics(self) -> [str]:
+        return [SysCommandTopic.SYS_INFO]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        infos = dict(psutil.virtual_memory()._asdict())
+        if pynvml:
+            try:
+                pynvml.nvmlInit()
+                device_count = pynvml.nvmlDeviceGetCount()
+                gpu_info = {"gpu_count": device_count}
+                for index in range(device_count):
+                    handle = pynvml.nvmlDeviceGetHandleByIndex(index)
+                    gpu_info[f"gpu_device_{index}"] = pynvml.nvmlDeviceGetName(handle).decode("utf-8")
+                pynvml.nvmlShutdown()
+                infos.update(gpu_info)
+            except pynvml.nvml.NVMLError_LibraryNotFound:
+                pass
+
+        # docker_image_tag = os.environ.get('DOCKER_IMAGE_TAG', 'N/A')
+        # infos.update({'docker_image_tag':docker_image_tag})
+        message = Message(topic="reply_" + req.topic, body=json.dumps(infos))
+        print("return sys_info")
+        print(infos)
+        return message
diff --git a/nvflare/private/fed/client/training_cmds.py b/nvflare/private/fed/client/training_cmds.py
new file mode 100644
index 0000000000..20a79914e5
--- /dev/null
+++ b/nvflare/private/fed/client/training_cmds.py
@@ -0,0 +1,161 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from typing import List
+
+from nvflare.private.admin_defs import Message
+from nvflare.private.defs import RequestHeader, TrainingTopic
+from nvflare.private.fed.client.client_engine_internal_spec import ClientEngineInternalSpec
+
+from .admin import RequestProcessor
+
+
+class StartAppProcessor(RequestProcessor):
+    def get_topics(self) -> List[str]:
+        return [TrainingTopic.START]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+
+        run_number = int(req.get_header(RequestHeader.RUN_NUM))
+        result = engine.start_app(run_number)
+        if not result:
+            result = "OK"
+        return Message(topic="reply_" + req.topic, body=result)
+
+
+class AbortAppProcessor(RequestProcessor):
+    def get_topics(self) -> List[str]:
+        return [TrainingTopic.ABORT]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+        run_number = int(req.get_header(RequestHeader.RUN_NUM))
+        result = engine.abort_app(run_number)
+        if not result:
+            result = "OK"
+        return Message(topic="reply_" + req.topic, body=result)
+
+
+class AbortTaskProcessor(RequestProcessor):
+    def get_topics(self) -> List[str]:
+        return [TrainingTopic.ABORT_TASK]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+        run_number = int(req.get_header(RequestHeader.RUN_NUM))
+        result = engine.abort_task(run_number)
+        if not result:
+            result = "OK"
+        return Message(topic="reply_" + req.topic, body=result)
+
+
+class ShutdownClientProcessor(RequestProcessor):
+    def get_topics(self) -> List[str]:
+        return [TrainingTopic.SHUTDOWN]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+        result = engine.shutdown()
+        if not result:
+            result = "OK"
+        return Message(topic="reply_" + req.topic, body=result)
+
+
+class RestartClientProcessor(RequestProcessor):
+    def get_topics(self) -> List[str]:
+        return [TrainingTopic.RESTART]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+        result = engine.restart()
+        if not result:
+            result = "OK"
+        return Message(topic="reply_" + req.topic, body=result)
+
+
+class DeployProcessor(RequestProcessor):
+    def get_topics(self) -> List[str]:
+        return [TrainingTopic.DEPLOY]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+        run_number = req.get_header(RequestHeader.RUN_NUM)
+        app_name = req.get_header(RequestHeader.APP_NAME)
+        client_name = engine.get_client_name()
+        result = engine.deploy_app(app_name=app_name, run_num=run_number, client_name=client_name, app_data=req.body)
+        if not result:
+            result = "OK"
+        return Message(topic="reply_" + req.topic, body=result)
+
+
+class DeleteRunNumberProcessor(RequestProcessor):
+    def get_topics(self) -> List[str]:
+        return [TrainingTopic.DELETE_RUN]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+        run_number = int(req.get_header(RequestHeader.RUN_NUM))
+        result = engine.delete_run(run_number)
+        if not result:
+            result = "OK"
+        message = Message(topic="reply_" + req.topic, body=result)
+        return message
+
+
+class ClientStatusProcessor(RequestProcessor):
+    def get_topics(self) -> List[str]:
+        return [TrainingTopic.CHECK_STATUS]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+        result = engine.get_engine_status()
+        # run_info = engine.get_current_run_info()
+        # if not run_info or run_info.run_number < 0:
+        #     result = {
+        #         ClientStatusKey.RUN_NUM: 'none',
+        #         ClientStatusKey.CURRENT_TASK: 'none'
+        #     }
+        # else:
+        #     result = {
+        #         ClientStatusKey.RUN_NUM: str(run_info.run_number),
+        #         ClientStatusKey.CURRENT_TASK: run_info.current_task_name
+        #     }
+        result = json.dumps(result)
+        message = Message(topic="reply_" + req.topic, body=result)
+        return message
+
+
+class SetRunNumberProcessor(RequestProcessor):
+    def get_topics(self) -> List[str]:
+        return [TrainingTopic.SET_RUN_NUMBER]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        engine = app_ctx
+        assert isinstance(engine, ClientEngineInternalSpec)
+
+        run_number = int(req.get_header(RequestHeader.RUN_NUM))
+        result = engine.set_run_number(run_number)
+        if not result:
+            result = 'OK'
+        return Message(topic="reply_" + req.topic, body=result)
diff --git a/nvflare/private/fed/client/validation_cmd.py b/nvflare/private/fed/client/validation_cmd.py
new file mode 100644
index 0000000000..b238add5e5
--- /dev/null
+++ b/nvflare/private/fed/client/validation_cmd.py
@@ -0,0 +1,29 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.private.admin_defs import Message
+
+from .admin import RequestProcessor
+
+
+class ValidateRequestProcessor(RequestProcessor):
+    def get_topics(self) -> [str]:
+        return ["validate"]
+
+    def process(self, req: Message, app_ctx) -> Message:
+        cai = app_ctx
+
+        result = cai.do_validate(req)
+        message = Message(topic="reply_" + req.topic, body=result)
+        return message
diff --git a/nvflare/private/fed/protos/__init__.py b/nvflare/private/fed/protos/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/private/fed/protos/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/private/fed/protos/admin.proto b/nvflare/private/fed/protos/admin.proto
new file mode 100644
index 0000000000..b8b770c353
--- /dev/null
+++ b/nvflare/private/fed/protos/admin.proto
@@ -0,0 +1,49 @@
+// services and messages for admin agent and server communication
+
+syntax = "proto3";
+
+package admin;
+
+//import "google/protobuf/timestamp.proto";
+//import "google/protobuf/struct.proto";
+
+// The admin communication interfaces
+service AdminCommunicating {
+    // client retrieve requests.
+    rpc Retrieve (Client) returns (Messages) {}
+    // client send reply to server
+    rpc SendReply (Reply) returns (Empty) {}
+    // client send process results to server
+    rpc SendResult (Reply) returns (Empty) {}
+}
+
+// List of Messages
+message Messages {
+    repeated Message message = 1;
+}
+
+// Message between server and client
+message Message {
+    string id = 1;
+    string topic = 2;
+    string body_type = 3;
+    bytes body = 4;
+    map<string, string> headers = 5;
+}
+
+// Empty message for no return required
+message Empty {
+
+}
+
+// Client
+message Client {
+    string client_name = 1;
+}
+
+// Reply
+message Reply {
+    string client_name = 1;
+    Message message = 2;
+}
+
diff --git a/nvflare/private/fed/protos/admin_pb2.py b/nvflare/private/fed/protos/admin_pb2.py
new file mode 100644
index 0000000000..b585564eb1
--- /dev/null
+++ b/nvflare/private/fed/protos/admin_pb2.py
@@ -0,0 +1,369 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: nvflare/private/fed/protos/admin.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='nvflare/private/fed/protos/admin.proto',
+  package='admin',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n&nvflare/private/fed/protos/admin.proto\x12\x05\x61\x64min\"+\n\x08Messages\x12\x1f\n\x07message\x18\x01 \x03(\x0b\x32\x0e.admin.Message\"\xa3\x01\n\x07Message\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x11\n\tbody_type\x18\x03 \x01(\t\x12\x0c\n\x04\x62ody\x18\x04 \x01(\x0c\x12,\n\x07headers\x18\x05 \x03(\x0b\x32\x1b.admin.Message.HeadersEntry\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x07\n\x05\x45mpty\"\x1d\n\x06\x43lient\x12\x13\n\x0b\x63lient_name\x18\x01 \x01(\t\"=\n\x05Reply\x12\x13\n\x0b\x63lient_name\x18\x01 \x01(\t\x12\x1f\n\x07message\x18\x02 \x01(\x0b\x32\x0e.admin.Message2\x99\x01\n\x12\x41\x64minCommunicating\x12,\n\x08Retrieve\x12\r.admin.Client\x1a\x0f.admin.Messages\"\x00\x12)\n\tSendReply\x12\x0c.admin.Reply\x1a\x0c.admin.Empty\"\x00\x12*\n\nSendResult\x12\x0c.admin.Reply\x1a\x0c.admin.Empty\"\x00\x62\x06proto3'
+)
+
+
+
+
+_MESSAGES = _descriptor.Descriptor(
+  name='Messages',
+  full_name='admin.Messages',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='message', full_name='admin.Messages.message', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=49,
+  serialized_end=92,
+)
+
+
+_MESSAGE_HEADERSENTRY = _descriptor.Descriptor(
+  name='HeadersEntry',
+  full_name='admin.Message.HeadersEntry',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='key', full_name='admin.Message.HeadersEntry.key', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='value', full_name='admin.Message.HeadersEntry.value', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=b'8\001',
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=212,
+  serialized_end=258,
+)
+
+_MESSAGE = _descriptor.Descriptor(
+  name='Message',
+  full_name='admin.Message',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='id', full_name='admin.Message.id', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='topic', full_name='admin.Message.topic', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='body_type', full_name='admin.Message.body_type', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='body', full_name='admin.Message.body', index=3,
+      number=4, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"",
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='headers', full_name='admin.Message.headers', index=4,
+      number=5, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[_MESSAGE_HEADERSENTRY, ],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=95,
+  serialized_end=258,
+)
+
+
+_EMPTY = _descriptor.Descriptor(
+  name='Empty',
+  full_name='admin.Empty',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=260,
+  serialized_end=267,
+)
+
+
+_CLIENT = _descriptor.Descriptor(
+  name='Client',
+  full_name='admin.Client',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='client_name', full_name='admin.Client.client_name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=269,
+  serialized_end=298,
+)
+
+
+_REPLY = _descriptor.Descriptor(
+  name='Reply',
+  full_name='admin.Reply',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='client_name', full_name='admin.Reply.client_name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='message', full_name='admin.Reply.message', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=300,
+  serialized_end=361,
+)
+
+_MESSAGES.fields_by_name['message'].message_type = _MESSAGE
+_MESSAGE_HEADERSENTRY.containing_type = _MESSAGE
+_MESSAGE.fields_by_name['headers'].message_type = _MESSAGE_HEADERSENTRY
+_REPLY.fields_by_name['message'].message_type = _MESSAGE
+DESCRIPTOR.message_types_by_name['Messages'] = _MESSAGES
+DESCRIPTOR.message_types_by_name['Message'] = _MESSAGE
+DESCRIPTOR.message_types_by_name['Empty'] = _EMPTY
+DESCRIPTOR.message_types_by_name['Client'] = _CLIENT
+DESCRIPTOR.message_types_by_name['Reply'] = _REPLY
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Messages = _reflection.GeneratedProtocolMessageType('Messages', (_message.Message,), {
+  'DESCRIPTOR' : _MESSAGES,
+  '__module__' : 'nvflare.private.fed.protos.admin_pb2'
+  # @@protoc_insertion_point(class_scope:admin.Messages)
+  })
+_sym_db.RegisterMessage(Messages)
+
+Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), {
+
+  'HeadersEntry' : _reflection.GeneratedProtocolMessageType('HeadersEntry', (_message.Message,), {
+    'DESCRIPTOR' : _MESSAGE_HEADERSENTRY,
+    '__module__' : 'nvflare.private.fed.protos.admin_pb2'
+    # @@protoc_insertion_point(class_scope:admin.Message.HeadersEntry)
+    })
+  ,
+  'DESCRIPTOR' : _MESSAGE,
+  '__module__' : 'nvflare.private.fed.protos.admin_pb2'
+  # @@protoc_insertion_point(class_scope:admin.Message)
+  })
+_sym_db.RegisterMessage(Message)
+_sym_db.RegisterMessage(Message.HeadersEntry)
+
+Empty = _reflection.GeneratedProtocolMessageType('Empty', (_message.Message,), {
+  'DESCRIPTOR' : _EMPTY,
+  '__module__' : 'nvflare.private.fed.protos.admin_pb2'
+  # @@protoc_insertion_point(class_scope:admin.Empty)
+  })
+_sym_db.RegisterMessage(Empty)
+
+Client = _reflection.GeneratedProtocolMessageType('Client', (_message.Message,), {
+  'DESCRIPTOR' : _CLIENT,
+  '__module__' : 'nvflare.private.fed.protos.admin_pb2'
+  # @@protoc_insertion_point(class_scope:admin.Client)
+  })
+_sym_db.RegisterMessage(Client)
+
+Reply = _reflection.GeneratedProtocolMessageType('Reply', (_message.Message,), {
+  'DESCRIPTOR' : _REPLY,
+  '__module__' : 'nvflare.private.fed.protos.admin_pb2'
+  # @@protoc_insertion_point(class_scope:admin.Reply)
+  })
+_sym_db.RegisterMessage(Reply)
+
+
+_MESSAGE_HEADERSENTRY._options = None
+
+_ADMINCOMMUNICATING = _descriptor.ServiceDescriptor(
+  name='AdminCommunicating',
+  full_name='admin.AdminCommunicating',
+  file=DESCRIPTOR,
+  index=0,
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_start=364,
+  serialized_end=517,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='Retrieve',
+    full_name='admin.AdminCommunicating.Retrieve',
+    index=0,
+    containing_service=None,
+    input_type=_CLIENT,
+    output_type=_MESSAGES,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='SendReply',
+    full_name='admin.AdminCommunicating.SendReply',
+    index=1,
+    containing_service=None,
+    input_type=_REPLY,
+    output_type=_EMPTY,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='SendResult',
+    full_name='admin.AdminCommunicating.SendResult',
+    index=2,
+    containing_service=None,
+    input_type=_REPLY,
+    output_type=_EMPTY,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_ADMINCOMMUNICATING)
+
+DESCRIPTOR.services_by_name['AdminCommunicating'] = _ADMINCOMMUNICATING
+
+# @@protoc_insertion_point(module_scope)
diff --git a/nvflare/private/fed/protos/admin_pb2_grpc.py b/nvflare/private/fed/protos/admin_pb2_grpc.py
new file mode 100644
index 0000000000..df8d6691b6
--- /dev/null
+++ b/nvflare/private/fed/protos/admin_pb2_grpc.py
@@ -0,0 +1,161 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from nvflare.private.fed.protos import admin_pb2 as nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2
+
+
+class AdminCommunicatingStub(object):
+    """import "google/protobuf/timestamp.proto";
+    import "google/protobuf/struct.proto";
+
+    The admin communication interfaces
+    """
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Retrieve = channel.unary_unary(
+                '/admin.AdminCommunicating/Retrieve',
+                request_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Client.SerializeToString,
+                response_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Messages.FromString,
+                )
+        self.SendReply = channel.unary_unary(
+                '/admin.AdminCommunicating/SendReply',
+                request_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Reply.SerializeToString,
+                response_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Empty.FromString,
+                )
+        self.SendResult = channel.unary_unary(
+                '/admin.AdminCommunicating/SendResult',
+                request_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Reply.SerializeToString,
+                response_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Empty.FromString,
+                )
+
+
+class AdminCommunicatingServicer(object):
+    """import "google/protobuf/timestamp.proto";
+    import "google/protobuf/struct.proto";
+
+    The admin communication interfaces
+    """
+
+    def Retrieve(self, request, context):
+        """client retrieve requests.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def SendReply(self, request, context):
+        """client send reply to server
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def SendResult(self, request, context):
+        """client send process results to server
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_AdminCommunicatingServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'Retrieve': grpc.unary_unary_rpc_method_handler(
+                    servicer.Retrieve,
+                    request_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Client.FromString,
+                    response_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Messages.SerializeToString,
+            ),
+            'SendReply': grpc.unary_unary_rpc_method_handler(
+                    servicer.SendReply,
+                    request_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Reply.FromString,
+                    response_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Empty.SerializeToString,
+            ),
+            'SendResult': grpc.unary_unary_rpc_method_handler(
+                    servicer.SendResult,
+                    request_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Reply.FromString,
+                    response_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Empty.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'admin.AdminCommunicating', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class AdminCommunicating(object):
+    """import "google/protobuf/timestamp.proto";
+    import "google/protobuf/struct.proto";
+
+    The admin communication interfaces
+    """
+
+    @staticmethod
+    def Retrieve(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/admin.AdminCommunicating/Retrieve',
+            nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Client.SerializeToString,
+            nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Messages.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def SendReply(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/admin.AdminCommunicating/SendReply',
+            nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Reply.SerializeToString,
+            nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Empty.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def SendResult(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/admin.AdminCommunicating/SendResult',
+            nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Reply.SerializeToString,
+            nvflare_dot_private_dot_fed_dot_protos_dot_admin__pb2.Empty.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
diff --git a/nvflare/private/fed/protos/federated.proto b/nvflare/private/fed/protos/federated.proto
new file mode 100644
index 0000000000..5637b9a073
--- /dev/null
+++ b/nvflare/private/fed/protos/federated.proto
@@ -0,0 +1,149 @@
+// services and messages for client and model training server communication
+
+syntax = "proto3";
+
+package fedlearn;
+
+// import "tensorflow/core/framework/tensor.proto";
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/struct.proto";
+
+// The global federated model interfaces
+service FederatedTraining {
+    // client registration, so that it will contribute to the training
+    rpc Register (ClientLogin) returns (FederatedSummary) {}
+    // client quiting the federated training
+    rpc Quit (ClientState) returns (FederatedSummary) {}
+    // server to client model sharing
+    rpc GetTask (ClientState) returns (CurrentTask) {}
+    // client to server contribution submission
+    rpc SubmitUpdate (Contribution) returns (FederatedSummary) {}
+    // client to server heartbeat keep live
+    rpc Heartbeat (Token) returns (FederatedSummary) {}
+    // client to server aux channel communication
+    rpc AuxCommunicate (AuxMessage) returns (AuxReply) {}
+}
+
+// The current federated model's meta data
+message MetaData {
+    message Project {
+        // the server's task name, e.g. BraTS segmentation
+        string name = 1;
+        string version = 2;
+    }
+    // unique task identifier for the server
+    Project project = 1;
+    // the current global step of the federated model
+    int64 current_round = 2;
+    // the total global step of the federated model
+    int64 num_rounds = 3;
+    // current run_number
+    int64 run_number = 4;
+    // time stamp when the model is shared
+    google.protobuf.Timestamp created = 5;
+}
+
+// A model consists of multiple tensors
+message ModelData {
+    map<string, NDArray> params = 1;
+}
+
+// NDArray data for protobuf
+message NDArray {
+  bytes ndarray = 1;
+}
+
+//////////////////////////////////////////////////
+// Server specific message (created by the server)
+/////////////////////////////////////////////////
+
+// The server's response of the current model parameters
+message CurrentTask {
+    // Model meta data for versioning of the current model
+    MetaData meta = 1;
+    // The server's serialised deep learning model parameters
+    ModelData data = 2;
+    // Represents a structured value.
+    google.protobuf.Struct meta_data = 3;
+    // Execute task name
+    string task_name = 4;
+}
+
+// The server's summary of the model (for client's debugging purpose)
+message FederatedSummary {
+    MetaData meta = 1;
+    // Additional details from server to clients
+    string comment = 2;
+    // The client token
+    string token = 3;
+}
+
+
+//////////////////////////////////////////////////
+// Client specific message (created by the client)
+/////////////////////////////////////////////////
+
+// The client login information
+message ClientLogin {
+    // The client identification
+    string client_name = 1;
+    // The client login token
+    string token = 2;
+    // The client running IP
+    string client_ip = 3;
+    // The client's model meta
+    MetaData meta = 4;
+}
+
+// The client's request to join the federated round
+message ClientState {
+    // The client's model meta
+    MetaData meta = 1;
+    // The client's identifier
+    string client_name = 2;
+    // The client's token
+    string token = 3;
+    // Represents a structured value.
+    google.protobuf.Struct meta_data = 4;
+    // The context data.
+    map<string, NDArray> context = 5;
+}
+
+// The client's contribution to the current federated model
+message Contribution {
+    // The client's identity information
+    ClientState client = 1;
+    // number of location iterations used to compute this contribution
+    int64 n_iter = 2;
+    // type of the contribution, could be w or delta w
+    string type = 3;
+    // The contribution data (grads. or model params.) to be aggregated
+    ModelData data = 4;
+    // Represents a structured value.
+    google.protobuf.Struct meta_data = 5;
+    // Execute task name
+    string task_name = 6;
+}
+
+// The client's heartbeat to keep the client live
+message Token {
+    // The client's token
+    string token = 1;
+    // The client identification
+    string client_name = 2;
+}
+
+// The client's Aux communication message
+message AuxMessage {
+    // The client's identity information
+    ClientState client = 1;
+    // The Aux data.
+    map<string, NDArray> data = 2;
+}
+
+// The client's Aux communication message
+message AuxReply {
+    // The Aux reply data.
+    ModelData data = 1;
+}
+
diff --git a/nvflare/private/fed/protos/federated_pb2.py b/nvflare/private/fed/protos/federated_pb2.py
new file mode 100644
index 0000000000..0a8f65795a
--- /dev/null
+++ b/nvflare/private/fed/protos/federated_pb2.py
@@ -0,0 +1,935 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: nvflare/private/fed/protos/federated.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='nvflare/private/fed/protos/federated.proto',
+  package='fedlearn',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n*nvflare/private/fed/protos/federated.proto\x12\x08\x66\x65\x64learn\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xcd\x01\n\x08MetaData\x12+\n\x07project\x18\x01 \x01(\x0b\x32\x1a.fedlearn.MetaData.Project\x12\x15\n\rcurrent_round\x18\x02 \x01(\x03\x12\x12\n\nnum_rounds\x18\x03 \x01(\x03\x12\x12\n\nrun_number\x18\x04 \x01(\x03\x12+\n\x07\x63reated\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a(\n\x07Project\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"~\n\tModelData\x12/\n\x06params\x18\x01 \x03(\x0b\x32\x1f.fedlearn.ModelData.ParamsEntry\x1a@\n\x0bParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.fedlearn.NDArray:\x02\x38\x01\"\x1a\n\x07NDArray\x12\x0f\n\x07ndarray\x18\x01 \x01(\x0c\"\x91\x01\n\x0b\x43urrentTask\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.fedlearn.MetaData\x12!\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x13.fedlearn.ModelData\x12*\n\tmeta_data\x18\x03 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x11\n\ttask_name\x18\x04 \x01(\t\"T\n\x10\x46\x65\x64\x65ratedSummary\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.fedlearn.MetaData\x12\x0f\n\x07\x63omment\x18\x02 \x01(\t\x12\r\n\x05token\x18\x03 \x01(\t\"f\n\x0b\x43lientLogin\x12\x13\n\x0b\x63lient_name\x18\x01 \x01(\t\x12\r\n\x05token\x18\x02 \x01(\t\x12\x11\n\tclient_ip\x18\x03 \x01(\t\x12 \n\x04meta\x18\x04 \x01(\x0b\x32\x12.fedlearn.MetaData\"\xf7\x01\n\x0b\x43lientState\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.fedlearn.MetaData\x12\x13\n\x0b\x63lient_name\x18\x02 \x01(\t\x12\r\n\x05token\x18\x03 \x01(\t\x12*\n\tmeta_data\x18\x04 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x33\n\x07\x63ontext\x18\x05 \x03(\x0b\x32\".fedlearn.ClientState.ContextEntry\x1a\x41\n\x0c\x43ontextEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.fedlearn.NDArray:\x02\x38\x01\"\xb5\x01\n\x0c\x43ontribution\x12%\n\x06\x63lient\x18\x01 \x01(\x0b\x32\x15.fedlearn.ClientState\x12\x0e\n\x06n_iter\x18\x02 \x01(\x03\x12\x0c\n\x04type\x18\x03 \x01(\t\x12!\n\x04\x64\x61ta\x18\x04 \x01(\x0b\x32\x13.fedlearn.ModelData\x12*\n\tmeta_data\x18\x05 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x11\n\ttask_name\x18\x06 \x01(\t\"+\n\x05Token\x12\r\n\x05token\x18\x01 \x01(\t\x12\x13\n\x0b\x63lient_name\x18\x02 \x01(\t\"\xa1\x01\n\nAuxMessage\x12%\n\x06\x63lient\x18\x01 \x01(\x0b\x32\x15.fedlearn.ClientState\x12,\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32\x1e.fedlearn.AuxMessage.DataEntry\x1a>\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.fedlearn.NDArray:\x02\x38\x01\"-\n\x08\x41uxReply\x12!\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32\x13.fedlearn.ModelData2\x8c\x03\n\x11\x46\x65\x64\x65ratedTraining\x12?\n\x08Register\x12\x15.fedlearn.ClientLogin\x1a\x1a.fedlearn.FederatedSummary\"\x00\x12;\n\x04Quit\x12\x15.fedlearn.ClientState\x1a\x1a.fedlearn.FederatedSummary\"\x00\x12\x39\n\x07GetTask\x12\x15.fedlearn.ClientState\x1a\x15.fedlearn.CurrentTask\"\x00\x12\x44\n\x0cSubmitUpdate\x12\x16.fedlearn.Contribution\x1a\x1a.fedlearn.FederatedSummary\"\x00\x12:\n\tHeartbeat\x12\x0f.fedlearn.Token\x1a\x1a.fedlearn.FederatedSummary\"\x00\x12<\n\x0e\x41uxCommunicate\x12\x14.fedlearn.AuxMessage\x1a\x12.fedlearn.AuxReply\"\x00\x62\x06proto3'
+  ,
+  dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,])
+
+
+
+
+_METADATA_PROJECT = _descriptor.Descriptor(
+  name='Project',
+  full_name='fedlearn.MetaData.Project',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='name', full_name='fedlearn.MetaData.Project.name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='version', full_name='fedlearn.MetaData.Project.version', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=285,
+  serialized_end=325,
+)
+
+_METADATA = _descriptor.Descriptor(
+  name='MetaData',
+  full_name='fedlearn.MetaData',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='project', full_name='fedlearn.MetaData.project', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='current_round', full_name='fedlearn.MetaData.current_round', index=1,
+      number=2, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='num_rounds', full_name='fedlearn.MetaData.num_rounds', index=2,
+      number=3, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='run_number', full_name='fedlearn.MetaData.run_number', index=3,
+      number=4, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='created', full_name='fedlearn.MetaData.created', index=4,
+      number=5, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[_METADATA_PROJECT, ],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=120,
+  serialized_end=325,
+)
+
+
+_MODELDATA_PARAMSENTRY = _descriptor.Descriptor(
+  name='ParamsEntry',
+  full_name='fedlearn.ModelData.ParamsEntry',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='key', full_name='fedlearn.ModelData.ParamsEntry.key', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='value', full_name='fedlearn.ModelData.ParamsEntry.value', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=b'8\001',
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=389,
+  serialized_end=453,
+)
+
+_MODELDATA = _descriptor.Descriptor(
+  name='ModelData',
+  full_name='fedlearn.ModelData',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='params', full_name='fedlearn.ModelData.params', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[_MODELDATA_PARAMSENTRY, ],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=327,
+  serialized_end=453,
+)
+
+
+_NDARRAY = _descriptor.Descriptor(
+  name='NDArray',
+  full_name='fedlearn.NDArray',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='ndarray', full_name='fedlearn.NDArray.ndarray', index=0,
+      number=1, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"",
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=455,
+  serialized_end=481,
+)
+
+
+_CURRENTTASK = _descriptor.Descriptor(
+  name='CurrentTask',
+  full_name='fedlearn.CurrentTask',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='meta', full_name='fedlearn.CurrentTask.meta', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='data', full_name='fedlearn.CurrentTask.data', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='meta_data', full_name='fedlearn.CurrentTask.meta_data', index=2,
+      number=3, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='task_name', full_name='fedlearn.CurrentTask.task_name', index=3,
+      number=4, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=484,
+  serialized_end=629,
+)
+
+
+_FEDERATEDSUMMARY = _descriptor.Descriptor(
+  name='FederatedSummary',
+  full_name='fedlearn.FederatedSummary',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='meta', full_name='fedlearn.FederatedSummary.meta', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='comment', full_name='fedlearn.FederatedSummary.comment', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='token', full_name='fedlearn.FederatedSummary.token', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=631,
+  serialized_end=715,
+)
+
+
+_CLIENTLOGIN = _descriptor.Descriptor(
+  name='ClientLogin',
+  full_name='fedlearn.ClientLogin',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='client_name', full_name='fedlearn.ClientLogin.client_name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='token', full_name='fedlearn.ClientLogin.token', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='client_ip', full_name='fedlearn.ClientLogin.client_ip', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='meta', full_name='fedlearn.ClientLogin.meta', index=3,
+      number=4, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=717,
+  serialized_end=819,
+)
+
+
+_CLIENTSTATE_CONTEXTENTRY = _descriptor.Descriptor(
+  name='ContextEntry',
+  full_name='fedlearn.ClientState.ContextEntry',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='key', full_name='fedlearn.ClientState.ContextEntry.key', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='value', full_name='fedlearn.ClientState.ContextEntry.value', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=b'8\001',
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=1004,
+  serialized_end=1069,
+)
+
+_CLIENTSTATE = _descriptor.Descriptor(
+  name='ClientState',
+  full_name='fedlearn.ClientState',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='meta', full_name='fedlearn.ClientState.meta', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='client_name', full_name='fedlearn.ClientState.client_name', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='token', full_name='fedlearn.ClientState.token', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='meta_data', full_name='fedlearn.ClientState.meta_data', index=3,
+      number=4, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='context', full_name='fedlearn.ClientState.context', index=4,
+      number=5, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[_CLIENTSTATE_CONTEXTENTRY, ],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=822,
+  serialized_end=1069,
+)
+
+
+_CONTRIBUTION = _descriptor.Descriptor(
+  name='Contribution',
+  full_name='fedlearn.Contribution',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='client', full_name='fedlearn.Contribution.client', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='n_iter', full_name='fedlearn.Contribution.n_iter', index=1,
+      number=2, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='type', full_name='fedlearn.Contribution.type', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='data', full_name='fedlearn.Contribution.data', index=3,
+      number=4, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='meta_data', full_name='fedlearn.Contribution.meta_data', index=4,
+      number=5, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='task_name', full_name='fedlearn.Contribution.task_name', index=5,
+      number=6, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=1072,
+  serialized_end=1253,
+)
+
+
+_TOKEN = _descriptor.Descriptor(
+  name='Token',
+  full_name='fedlearn.Token',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='token', full_name='fedlearn.Token.token', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='client_name', full_name='fedlearn.Token.client_name', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=1255,
+  serialized_end=1298,
+)
+
+
+_AUXMESSAGE_DATAENTRY = _descriptor.Descriptor(
+  name='DataEntry',
+  full_name='fedlearn.AuxMessage.DataEntry',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='key', full_name='fedlearn.AuxMessage.DataEntry.key', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='value', full_name='fedlearn.AuxMessage.DataEntry.value', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=b'8\001',
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=1400,
+  serialized_end=1462,
+)
+
+_AUXMESSAGE = _descriptor.Descriptor(
+  name='AuxMessage',
+  full_name='fedlearn.AuxMessage',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='client', full_name='fedlearn.AuxMessage.client', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='data', full_name='fedlearn.AuxMessage.data', index=1,
+      number=2, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[_AUXMESSAGE_DATAENTRY, ],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=1301,
+  serialized_end=1462,
+)
+
+
+_AUXREPLY = _descriptor.Descriptor(
+  name='AuxReply',
+  full_name='fedlearn.AuxReply',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='data', full_name='fedlearn.AuxReply.data', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=1464,
+  serialized_end=1509,
+)
+
+_METADATA_PROJECT.containing_type = _METADATA
+_METADATA.fields_by_name['project'].message_type = _METADATA_PROJECT
+_METADATA.fields_by_name['created'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP
+_MODELDATA_PARAMSENTRY.fields_by_name['value'].message_type = _NDARRAY
+_MODELDATA_PARAMSENTRY.containing_type = _MODELDATA
+_MODELDATA.fields_by_name['params'].message_type = _MODELDATA_PARAMSENTRY
+_CURRENTTASK.fields_by_name['meta'].message_type = _METADATA
+_CURRENTTASK.fields_by_name['data'].message_type = _MODELDATA
+_CURRENTTASK.fields_by_name['meta_data'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT
+_FEDERATEDSUMMARY.fields_by_name['meta'].message_type = _METADATA
+_CLIENTLOGIN.fields_by_name['meta'].message_type = _METADATA
+_CLIENTSTATE_CONTEXTENTRY.fields_by_name['value'].message_type = _NDARRAY
+_CLIENTSTATE_CONTEXTENTRY.containing_type = _CLIENTSTATE
+_CLIENTSTATE.fields_by_name['meta'].message_type = _METADATA
+_CLIENTSTATE.fields_by_name['meta_data'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT
+_CLIENTSTATE.fields_by_name['context'].message_type = _CLIENTSTATE_CONTEXTENTRY
+_CONTRIBUTION.fields_by_name['client'].message_type = _CLIENTSTATE
+_CONTRIBUTION.fields_by_name['data'].message_type = _MODELDATA
+_CONTRIBUTION.fields_by_name['meta_data'].message_type = google_dot_protobuf_dot_struct__pb2._STRUCT
+_AUXMESSAGE_DATAENTRY.fields_by_name['value'].message_type = _NDARRAY
+_AUXMESSAGE_DATAENTRY.containing_type = _AUXMESSAGE
+_AUXMESSAGE.fields_by_name['client'].message_type = _CLIENTSTATE
+_AUXMESSAGE.fields_by_name['data'].message_type = _AUXMESSAGE_DATAENTRY
+_AUXREPLY.fields_by_name['data'].message_type = _MODELDATA
+DESCRIPTOR.message_types_by_name['MetaData'] = _METADATA
+DESCRIPTOR.message_types_by_name['ModelData'] = _MODELDATA
+DESCRIPTOR.message_types_by_name['NDArray'] = _NDARRAY
+DESCRIPTOR.message_types_by_name['CurrentTask'] = _CURRENTTASK
+DESCRIPTOR.message_types_by_name['FederatedSummary'] = _FEDERATEDSUMMARY
+DESCRIPTOR.message_types_by_name['ClientLogin'] = _CLIENTLOGIN
+DESCRIPTOR.message_types_by_name['ClientState'] = _CLIENTSTATE
+DESCRIPTOR.message_types_by_name['Contribution'] = _CONTRIBUTION
+DESCRIPTOR.message_types_by_name['Token'] = _TOKEN
+DESCRIPTOR.message_types_by_name['AuxMessage'] = _AUXMESSAGE
+DESCRIPTOR.message_types_by_name['AuxReply'] = _AUXREPLY
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+MetaData = _reflection.GeneratedProtocolMessageType('MetaData', (_message.Message,), {
+
+  'Project' : _reflection.GeneratedProtocolMessageType('Project', (_message.Message,), {
+    'DESCRIPTOR' : _METADATA_PROJECT,
+    '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+    # @@protoc_insertion_point(class_scope:fedlearn.MetaData.Project)
+    })
+  ,
+  'DESCRIPTOR' : _METADATA,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.MetaData)
+  })
+_sym_db.RegisterMessage(MetaData)
+_sym_db.RegisterMessage(MetaData.Project)
+
+ModelData = _reflection.GeneratedProtocolMessageType('ModelData', (_message.Message,), {
+
+  'ParamsEntry' : _reflection.GeneratedProtocolMessageType('ParamsEntry', (_message.Message,), {
+    'DESCRIPTOR' : _MODELDATA_PARAMSENTRY,
+    '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+    # @@protoc_insertion_point(class_scope:fedlearn.ModelData.ParamsEntry)
+    })
+  ,
+  'DESCRIPTOR' : _MODELDATA,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.ModelData)
+  })
+_sym_db.RegisterMessage(ModelData)
+_sym_db.RegisterMessage(ModelData.ParamsEntry)
+
+NDArray = _reflection.GeneratedProtocolMessageType('NDArray', (_message.Message,), {
+  'DESCRIPTOR' : _NDARRAY,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.NDArray)
+  })
+_sym_db.RegisterMessage(NDArray)
+
+CurrentTask = _reflection.GeneratedProtocolMessageType('CurrentTask', (_message.Message,), {
+  'DESCRIPTOR' : _CURRENTTASK,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.CurrentTask)
+  })
+_sym_db.RegisterMessage(CurrentTask)
+
+FederatedSummary = _reflection.GeneratedProtocolMessageType('FederatedSummary', (_message.Message,), {
+  'DESCRIPTOR' : _FEDERATEDSUMMARY,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.FederatedSummary)
+  })
+_sym_db.RegisterMessage(FederatedSummary)
+
+ClientLogin = _reflection.GeneratedProtocolMessageType('ClientLogin', (_message.Message,), {
+  'DESCRIPTOR' : _CLIENTLOGIN,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.ClientLogin)
+  })
+_sym_db.RegisterMessage(ClientLogin)
+
+ClientState = _reflection.GeneratedProtocolMessageType('ClientState', (_message.Message,), {
+
+  'ContextEntry' : _reflection.GeneratedProtocolMessageType('ContextEntry', (_message.Message,), {
+    'DESCRIPTOR' : _CLIENTSTATE_CONTEXTENTRY,
+    '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+    # @@protoc_insertion_point(class_scope:fedlearn.ClientState.ContextEntry)
+    })
+  ,
+  'DESCRIPTOR' : _CLIENTSTATE,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.ClientState)
+  })
+_sym_db.RegisterMessage(ClientState)
+_sym_db.RegisterMessage(ClientState.ContextEntry)
+
+Contribution = _reflection.GeneratedProtocolMessageType('Contribution', (_message.Message,), {
+  'DESCRIPTOR' : _CONTRIBUTION,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.Contribution)
+  })
+_sym_db.RegisterMessage(Contribution)
+
+Token = _reflection.GeneratedProtocolMessageType('Token', (_message.Message,), {
+  'DESCRIPTOR' : _TOKEN,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.Token)
+  })
+_sym_db.RegisterMessage(Token)
+
+AuxMessage = _reflection.GeneratedProtocolMessageType('AuxMessage', (_message.Message,), {
+
+  'DataEntry' : _reflection.GeneratedProtocolMessageType('DataEntry', (_message.Message,), {
+    'DESCRIPTOR' : _AUXMESSAGE_DATAENTRY,
+    '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+    # @@protoc_insertion_point(class_scope:fedlearn.AuxMessage.DataEntry)
+    })
+  ,
+  'DESCRIPTOR' : _AUXMESSAGE,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.AuxMessage)
+  })
+_sym_db.RegisterMessage(AuxMessage)
+_sym_db.RegisterMessage(AuxMessage.DataEntry)
+
+AuxReply = _reflection.GeneratedProtocolMessageType('AuxReply', (_message.Message,), {
+  'DESCRIPTOR' : _AUXREPLY,
+  '__module__' : 'nvflare.private.fed.protos.federated_pb2'
+  # @@protoc_insertion_point(class_scope:fedlearn.AuxReply)
+  })
+_sym_db.RegisterMessage(AuxReply)
+
+
+_MODELDATA_PARAMSENTRY._options = None
+_CLIENTSTATE_CONTEXTENTRY._options = None
+_AUXMESSAGE_DATAENTRY._options = None
+
+_FEDERATEDTRAINING = _descriptor.ServiceDescriptor(
+  name='FederatedTraining',
+  full_name='fedlearn.FederatedTraining',
+  file=DESCRIPTOR,
+  index=0,
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_start=1512,
+  serialized_end=1908,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='Register',
+    full_name='fedlearn.FederatedTraining.Register',
+    index=0,
+    containing_service=None,
+    input_type=_CLIENTLOGIN,
+    output_type=_FEDERATEDSUMMARY,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='Quit',
+    full_name='fedlearn.FederatedTraining.Quit',
+    index=1,
+    containing_service=None,
+    input_type=_CLIENTSTATE,
+    output_type=_FEDERATEDSUMMARY,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='GetTask',
+    full_name='fedlearn.FederatedTraining.GetTask',
+    index=2,
+    containing_service=None,
+    input_type=_CLIENTSTATE,
+    output_type=_CURRENTTASK,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='SubmitUpdate',
+    full_name='fedlearn.FederatedTraining.SubmitUpdate',
+    index=3,
+    containing_service=None,
+    input_type=_CONTRIBUTION,
+    output_type=_FEDERATEDSUMMARY,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='Heartbeat',
+    full_name='fedlearn.FederatedTraining.Heartbeat',
+    index=4,
+    containing_service=None,
+    input_type=_TOKEN,
+    output_type=_FEDERATEDSUMMARY,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='AuxCommunicate',
+    full_name='fedlearn.FederatedTraining.AuxCommunicate',
+    index=5,
+    containing_service=None,
+    input_type=_AUXMESSAGE,
+    output_type=_AUXREPLY,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_FEDERATEDTRAINING)
+
+DESCRIPTOR.services_by_name['FederatedTraining'] = _FEDERATEDTRAINING
+
+# @@protoc_insertion_point(module_scope)
diff --git a/nvflare/private/fed/protos/federated_pb2_grpc.py b/nvflare/private/fed/protos/federated_pb2_grpc.py
new file mode 100644
index 0000000000..cacbb9dac3
--- /dev/null
+++ b/nvflare/private/fed/protos/federated_pb2_grpc.py
@@ -0,0 +1,254 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from nvflare.private.fed.protos import federated_pb2 as nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2
+
+
+class FederatedTrainingStub(object):
+    """The global federated model interfaces
+    """
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Register = channel.unary_unary(
+                '/fedlearn.FederatedTraining/Register',
+                request_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.ClientLogin.SerializeToString,
+                response_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.FromString,
+                )
+        self.Quit = channel.unary_unary(
+                '/fedlearn.FederatedTraining/Quit',
+                request_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.ClientState.SerializeToString,
+                response_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.FromString,
+                )
+        self.GetTask = channel.unary_unary(
+                '/fedlearn.FederatedTraining/GetTask',
+                request_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.ClientState.SerializeToString,
+                response_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.CurrentTask.FromString,
+                )
+        self.SubmitUpdate = channel.unary_unary(
+                '/fedlearn.FederatedTraining/SubmitUpdate',
+                request_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.Contribution.SerializeToString,
+                response_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.FromString,
+                )
+        self.Heartbeat = channel.unary_unary(
+                '/fedlearn.FederatedTraining/Heartbeat',
+                request_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.Token.SerializeToString,
+                response_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.FromString,
+                )
+        self.AuxCommunicate = channel.unary_unary(
+                '/fedlearn.FederatedTraining/AuxCommunicate',
+                request_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.AuxMessage.SerializeToString,
+                response_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.AuxReply.FromString,
+                )
+
+
+class FederatedTrainingServicer(object):
+    """The global federated model interfaces
+    """
+
+    def Register(self, request, context):
+        """client registration, so that it will contribute to the training
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Quit(self, request, context):
+        """client quiting the federated training
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def GetTask(self, request, context):
+        """server to client model sharing
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def SubmitUpdate(self, request, context):
+        """client to server contribution submission
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Heartbeat(self, request, context):
+        """client to server heartbeat keep live
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def AuxCommunicate(self, request, context):
+        """client to server aux channel communication
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_FederatedTrainingServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'Register': grpc.unary_unary_rpc_method_handler(
+                    servicer.Register,
+                    request_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.ClientLogin.FromString,
+                    response_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.SerializeToString,
+            ),
+            'Quit': grpc.unary_unary_rpc_method_handler(
+                    servicer.Quit,
+                    request_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.ClientState.FromString,
+                    response_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.SerializeToString,
+            ),
+            'GetTask': grpc.unary_unary_rpc_method_handler(
+                    servicer.GetTask,
+                    request_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.ClientState.FromString,
+                    response_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.CurrentTask.SerializeToString,
+            ),
+            'SubmitUpdate': grpc.unary_unary_rpc_method_handler(
+                    servicer.SubmitUpdate,
+                    request_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.Contribution.FromString,
+                    response_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.SerializeToString,
+            ),
+            'Heartbeat': grpc.unary_unary_rpc_method_handler(
+                    servicer.Heartbeat,
+                    request_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.Token.FromString,
+                    response_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.SerializeToString,
+            ),
+            'AuxCommunicate': grpc.unary_unary_rpc_method_handler(
+                    servicer.AuxCommunicate,
+                    request_deserializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.AuxMessage.FromString,
+                    response_serializer=nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.AuxReply.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'fedlearn.FederatedTraining', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class FederatedTraining(object):
+    """The global federated model interfaces
+    """
+
+    @staticmethod
+    def Register(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/fedlearn.FederatedTraining/Register',
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.ClientLogin.SerializeToString,
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Quit(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/fedlearn.FederatedTraining/Quit',
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.ClientState.SerializeToString,
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def GetTask(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/fedlearn.FederatedTraining/GetTask',
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.ClientState.SerializeToString,
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.CurrentTask.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def SubmitUpdate(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/fedlearn.FederatedTraining/SubmitUpdate',
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.Contribution.SerializeToString,
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Heartbeat(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/fedlearn.FederatedTraining/Heartbeat',
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.Token.SerializeToString,
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.FederatedSummary.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def AuxCommunicate(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/fedlearn.FederatedTraining/AuxCommunicate',
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.AuxMessage.SerializeToString,
+            nvflare_dot_private_dot_fed_dot_protos_dot_federated__pb2.AuxReply.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
diff --git a/nvflare/private/fed/server/__init__.py b/nvflare/private/fed/server/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/private/fed/server/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/private/fed/server/admin.py b/nvflare/private/fed/server/admin.py
new file mode 100644
index 0000000000..375c2b3213
--- /dev/null
+++ b/nvflare/private/fed/server/admin.py
@@ -0,0 +1,458 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandModule
+from nvflare.fuel.hci.server.audit import CommandAudit
+from nvflare.fuel.hci.server.authz import AuthorizationService, AuthzCommandModule, AuthzFilter
+from nvflare.fuel.hci.server.builtin import new_command_register_with_builtin_module
+from nvflare.fuel.hci.server.constants import ConnProps
+from nvflare.fuel.hci.server.file_transfer import FileTransferModule
+from nvflare.fuel.hci.server.hci import AdminServer
+from nvflare.fuel.hci.server.login import LoginModule, SessionManager, SimpleAuthenticator
+from nvflare.fuel.sec.audit import Auditor, AuditService
+from nvflare.private.admin_defs import Message
+
+from .app_authz import AppAuthzService
+
+
+def new_message(conn: Connection, topic, body) -> Message:
+    msg = Message(topic=topic, body=body)
+    event_id = conn.get_prop(ConnProps.EVENT_ID, default=None)
+    if event_id:
+        msg.set_header(ConnProps.EVENT_ID, event_id)
+
+    user_name = conn.get_prop(ConnProps.USER_NAME, default=None)
+    if user_name:
+        msg.set_header(ConnProps.USER_NAME, user_name)
+
+    return msg
+
+
+class _Waiter(object):
+    def __init__(self, req: Message):
+        self.req = req
+        self.reply = None
+        self.reply_time = None
+
+
+class _Client(object):
+    def __init__(self, token):
+        self.token = token
+        self.last_heard_time = None
+        self.outgoing_reqs = []
+        self.fnf_reqs = []   # fire-and-forget requests
+        self.waiters = {}  # ref => waiter
+        self.req_lock = threading.Lock()
+        self.waiter_lock = threading.Lock()
+
+    def send(self, req: Message):
+        waiter = _Waiter(req)
+        with self.req_lock:
+            self.outgoing_reqs.append(req)
+
+        with self.waiter_lock:
+            self.waiters[req.id] = waiter
+
+        return waiter
+
+    def fire_and_forget(self, reqs: [Message]):
+        with self.req_lock:
+            for r in reqs:
+                self.fnf_reqs.append(r)
+
+    def get_outgoing_requests(self, max_num_reqs=0) -> [Message]:
+        result = []
+        with self.req_lock:
+            # reqs in outgoing_reqs have higher priority
+            q = self.outgoing_reqs
+            if len(self.outgoing_reqs) <= 0:
+                # no regular reqs - take fire-and-forget reqs
+                q = self.fnf_reqs
+
+            if max_num_reqs <= 0:
+                num_reqs = len(q)
+            else:
+                num_reqs = min(max_num_reqs, len(q))
+
+            for i in range(num_reqs):
+                result.append(q.pop(0))
+
+        return result
+
+    def accept_reply(self, reply: Message):
+        self.last_heard_time = time.time()
+        ref_id = reply.get_ref_id()
+        with self.waiter_lock:
+            # print("accept_reply waiter_lock ID: {}".format(ref_id))
+            w = self.waiters.pop(ref_id, None)
+            # print("accept_reply waiter_lock ID Done.....")
+
+        if w:
+            assert isinstance(w, _Waiter)
+            w.reply = reply
+            w.reply_time = time.time()
+
+    def cancel_waiter(self, waiter: _Waiter):
+        req = waiter.req
+        with self.waiter_lock:
+            # print("cancel waiter_lock ID: {}".format(req.id))
+            waiter = self.waiters.pop(req.id, None)
+            # print("cancel waiter_lock ID done .....")
+
+        if waiter:
+            with self.req_lock:
+                if req in self.outgoing_reqs:
+                    self.outgoing_reqs.remove(req)
+
+
+class ClientReply(object):
+    def __init__(self, client_token: str, req: Message, reply: Message):
+        self.client_token = client_token
+        self.request = req
+        self.reply = reply
+
+
+class _ClientReq(object):
+    def __init__(self, client, req: Message):
+        self.client = client
+        self.req = req
+        self.waiter = None
+
+
+class FedAdminServer(AdminServer):
+    """
+    The FedAdminServer is the framework for developing admin commands.
+
+    Args:
+        fed_admin_interface: the Fed Server's admin interface
+        users: a dict of user name => pwd hash
+        cmd_modules: a list of CommandModules
+        file_upload_dir: the directory for uploaded files
+        file_download_dir: the directory for files to be downloaded
+        allowed_shell_cmds: list of shell commands allowed. If not specified, all allowed.
+        host: the IP address of the admin server
+        port: port number of admin server
+        ca_cert_file_name: the root CA's cert file name
+        server_cert_file_name: server's cert, signed by the CA
+        server_key_file_name: server's private key file
+        accepted_client_cns: list of accepted Common Names from client, if specified
+    """
+
+    def __init__(
+        self,
+        fed_admin_interface,
+        users,
+        cmd_modules,
+        file_upload_dir,
+        file_download_dir,
+        allowed_shell_cmds,
+        host,
+        port,
+        ca_cert_file_name,
+        server_cert_file_name,
+        server_key_file_name,
+        accepted_client_cns=None,
+        app_validator=None,
+    ):
+        cmd_reg = new_command_register_with_builtin_module(app_ctx=fed_admin_interface)
+        self.sai = fed_admin_interface
+        self.allowed_shell_cmds = allowed_shell_cmds
+
+        authenticator = SimpleAuthenticator(users)
+        sess_mgr = SessionManager()
+        login_module = LoginModule(authenticator, sess_mgr)
+        cmd_reg.register_module(login_module)
+
+        # register filters - order is important!
+        # login_module is also a filter that determines user name or authenticated
+        cmd_reg.add_filter(login_module)
+
+        # next is the authorization filter and command module
+        authorizer = AuthorizationService.get_authorizer()
+        authz_filter = AuthzFilter(authorizer=authorizer)
+        cmd_reg.add_filter(authz_filter)
+        authz_cmd_module = AuthzCommandModule(authorizer=authorizer)
+        cmd_reg.register_module(authz_cmd_module)
+
+        # audit filter records commands to audit trail
+        auditor = AuditService.get_auditor()
+        assert isinstance(auditor, Auditor), "auditor must be Auditor but got {}".format(type(auditor))
+        audit_filter = CommandAudit(auditor)
+        cmd_reg.add_filter(audit_filter)
+
+        self.file_upload_dir = file_upload_dir
+        self.file_download_dir = file_download_dir
+
+        AppAuthzService.initialize(app_validator)
+        cmd_reg.register_module(
+            FileTransferModule(
+                upload_dir=file_upload_dir,
+                download_dir=file_download_dir,
+                upload_folder_authz_func=AppAuthzService.authorize_upload,
+            )
+        )
+
+        if cmd_modules:
+            assert isinstance(cmd_modules, list), "cmd_modules must be list"
+
+            for m in cmd_modules:
+                assert isinstance(m, CommandModule), "cmd_modules must contain CommandModule"
+                cmd_reg.register_module(m)
+
+        AdminServer.__init__(
+            self,
+            cmd_reg=cmd_reg,
+            host=host,
+            port=port,
+            ca_cert=ca_cert_file_name,
+            server_cert=server_cert_file_name,
+            server_key=server_key_file_name,
+            accepted_client_cns=accepted_client_cns,
+        )
+
+        self.clients = {}  # token => _Client
+        self.client_lock = threading.Lock()
+        self.timeout = 10.0  # default admin command timeout
+
+    def client_heartbeat(self, token):
+        """
+        This method is called by the Fed Engine to indicate the client is alive.
+        Args:
+            token:
+
+        Returns:
+
+        """
+        with self.client_lock:
+            client = self.clients.get(token)
+            if not client:
+                client = _Client(token)
+                self.clients[token] = client
+            client.last_heard_time = time.time()
+            return client
+
+    def client_dead(self, client_token):
+        """
+        This method is called by the Fed Engine to indicate the client is dead.
+
+        Args:
+            client_token: the session token of the client
+
+        Returns:
+
+        """
+        with self.client_lock:
+            self.clients.pop(client_token, None)
+
+    def get_client_tokens(self) -> []:
+        """
+        Get tokens of existing clients.
+        """
+        result = []
+        with self.client_lock:
+            for token in self.clients.keys():
+                result.append(token)
+        return result
+
+    def send_request_to_client(self, req: Message, client_token: str, timeout_secs=2.0) -> ClientReply:
+        assert isinstance(req, Message), "request must be Message"
+        reqs = {client_token: req}
+        replies = self.send_requests(reqs, timeout_secs)
+        if replies is None or len(replies) <= 0:
+            return None
+        else:
+            return replies[0]
+
+    def send_request_to_clients(self, req: Message, client_tokens: [str], timeout_secs=2.0) -> [ClientReply]:
+        assert isinstance(req, Message), "request must be Message"
+        reqs = {}
+        for token in client_tokens:
+            reqs[token] = req
+
+        return self.send_requests(reqs, timeout_secs)
+
+    def send_requests(self, requests: dict, timeout_secs=2.0) -> [ClientReply]:
+        """
+        This method is to be used by a Command Handler to send requests to Clients.
+        Hence it is run in the Command Handler's handling thread.
+        This is a blocking call - returned only after all responses are received or timeout.
+
+        Args:
+            requests: a dict of requests: client token => req message or list of msgs
+            timeout_secs: how long to wait for reply before timeout
+
+        Returns: a list of ClientReply
+
+        """
+
+        assert isinstance(requests, dict), "requests must be a dict"
+
+        if len(requests) <= 0:
+            return []
+
+        if timeout_secs <= 0.0:
+            # this is fire-and-forget!
+            for token, r in requests.items():
+                client = self.clients.get(token)
+                if not client:
+                    continue
+
+                assert isinstance(client, _Client)
+                if isinstance(r, list):
+                    reqs = r
+                else:
+                    reqs = [r]
+
+                client.fire_and_forget(reqs)
+            # No replies
+            return []
+
+        # Regular requests
+        client_reqs = []
+        with self.client_lock:
+            for token, r in requests.items():
+                client = self.clients.get(token)
+                if not client:
+                    continue
+
+                if isinstance(r, list):
+                    reqs = r
+                else:
+                    reqs = [r]
+
+                for req in reqs:
+                    assert isinstance(req, Message), "request must be a Message"
+                    client_reqs.append(_ClientReq(client, req))
+
+        return self._send_client_reqs(client_reqs, timeout_secs)
+
+    def _send_client_reqs(self, client_reqs, timeout_secs) -> [ClientReply]:
+        result = []
+        if len(client_reqs) <= 0:
+            return result
+
+        for cr in client_reqs:
+            cr.waiter = cr.client.send(cr.req)
+
+        start_time = time.time()
+        while True:
+            all_received = True
+            for cr in client_reqs:
+                if cr.waiter.reply_time is None:
+                    all_received = False
+                    break
+
+            if all_received:
+                break
+
+            if time.time() - start_time > timeout_secs:
+                # timeout
+                break
+
+            time.sleep(0.1)
+
+        for cr in client_reqs:
+            result.append(ClientReply(client_token=cr.client.token,
+                                      req=cr.waiter.req,
+                                      reply=cr.waiter.reply))
+
+            if cr.waiter.reply_time is None:
+                # this client timed out
+                cr.client.cancel_waiter(cr.waiter)
+
+        return result
+
+    def send_to_all_clients(self, req: Message, timeout_secs=2.0) -> [ClientReply]:
+        """
+        This method is to be used by a Command Handler to send a request to all Clients.
+        Hence it is run in the Command Handler's handling thread.
+        This is a blocking call - returned only after all responses are received or timeout.
+
+        Args:
+            req: the request to be sent
+            timeout_secs: how long to wait for reply before timeout
+
+        Returns: a list of ClientReply
+
+        """
+        assert isinstance(req, Message), "req must be a Message"
+        return self.send_requests_to_all_clients([req], timeout_secs)
+
+    def send_requests_to_all_clients(self, reqs: [Message], timeout_secs=2.0) -> [ClientReply]:
+        """
+        Send multiple request messages to all clients and wait for replies
+
+        Args:
+            reqs: requests to be sent
+            timeout_secs: how long to wait for reply before timeout
+
+        Returns: a list of ClientReply
+
+        """
+        if len(reqs) <= 0:
+            return []
+
+        client_reqs = []
+        with self.client_lock:
+            for _, client in self.clients.items():
+                for req in reqs:
+                    client_reqs.append(_ClientReq(client, req))
+
+        return self._send_client_reqs(client_reqs, timeout_secs)
+
+    def accept_reply(self, client_token, reply: Message):
+        """
+        This method is to be called by the FL Engine after a client response is received.
+        Hence it is called from the FL Engine's message processing thread.
+
+        Args:
+            client_token: session token of the client
+            reply: the reply message
+
+        Returns:
+        """
+        client = self.client_heartbeat(client_token)
+
+        ref_id = reply.get_ref_id()
+        assert ref_id is not None, "protocol error: missing ref_id in reply from client {}".format(client_token)
+
+        client.accept_reply(reply)
+
+    def get_outgoing_requests(self, client_token, max_reqs=0):
+        """
+        This method is called by FL Engine to get outgoing messages to the client, so it
+        can send them to the client.
+
+        Args:
+            client_token: session token of the client
+            max_reqs: max number of requests. 0 means unlimited.
+
+        Returns:
+
+        """
+        with self.client_lock:
+            client = self.clients.get(client_token)
+
+        if client:
+            return client.get_outgoing_requests(max_reqs)
+        else:
+            return []
+
+    def stop(self):
+        super().stop()
+        self.sai.close()
diff --git a/nvflare/private/fed/server/app_authz.py b/nvflare/private/fed/server/app_authz.py
new file mode 100644
index 0000000000..45026e55c0
--- /dev/null
+++ b/nvflare/private/fed/server/app_authz.py
@@ -0,0 +1,52 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.app_validation import AppValidationKey, AppValidator
+from nvflare.fuel.hci.server.authz import AuthzContext
+from nvflare.security.security import Action, FLAuthzContext
+
+
+class AppAuthzService(object):
+
+    app_validator = None
+
+    @staticmethod
+    def initialize(app_validator):
+        if app_validator and not isinstance(app_validator, AppValidator):
+            raise TypeError(f"app_validator must be an instance of AppValidator, but get {type(app_validator)}.")
+        AppAuthzService.app_validator = app_validator
+
+    @staticmethod
+    def _authorize_actions(app_path: str, sites: [str], actions) -> (str, AuthzContext):
+        if AppAuthzService.app_validator:
+            err, info = AppAuthzService.app_validator.validate(app_path)
+            if err:
+                return err, None
+
+            byoc = info.get(AppValidationKey.BYOC, False)
+            custom_datalist = info.get(AppValidationKey.CUSTOM_DATA_LIST, False)
+            if byoc:
+                actions.append(Action.BYOC)
+            if custom_datalist:
+                actions.append(Action.CUSTOM_DATALIST)
+
+        return "", FLAuthzContext.new_authz_context(site_names=sites, actions=actions)
+
+    @staticmethod
+    def authorize_upload(app_path: str) -> (str, AuthzContext):
+        return AppAuthzService._authorize_actions(app_path, ["server"], [Action.UPLOAD])
+
+    @staticmethod
+    def authorize_deploy(app_path: str, sites: [str]) -> (str, AuthzContext):
+        return AppAuthzService._authorize_actions(app_path, sites, [Action.DEPLOY])
diff --git a/nvflare/private/fed/server/client_manager.py b/nvflare/private/fed/server/client_manager.py
new file mode 100644
index 0000000000..918f334cd3
--- /dev/null
+++ b/nvflare/private/fed/server/client_manager.py
@@ -0,0 +1,201 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import threading
+import time
+import uuid
+
+import grpc
+
+from nvflare.apis.client import Client
+
+
+class ClientManager:
+    def __init__(self, project_name=None, min_num_clients=2, max_num_clients=10):
+        self.project_name = project_name
+        self.min_num_clients = min_num_clients
+        self.max_num_clients = max_num_clients
+        self.clients = dict()  # token => Client
+        self.lock = threading.Lock()
+
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def authenticate(self, request, context):
+        client = self.login_client(request, context)
+        if not client:
+            return None
+
+        client_ip = context.peer().split(":")[1]
+
+        if len(self.clients) >= self.max_num_clients:
+            context.abort(grpc.StatusCode.RESOURCE_EXHAUSTED, "Maximum number of clients reached")
+
+        # new client will join the current round immediately
+        with self.lock:
+            # self._set_instance_name(client)
+            self.clients.update({client.token: client})
+            self.logger.info(
+                "Client: New client {} joined. Sent token: {}.  Total clients: {}".format(
+                    request.client_name + "@" + client_ip, client.token, len(self.clients)
+                )
+            )
+        return client.token
+
+    def remove_client(self, token):
+        """
+        Remove a registered client through the token.
+        Args:
+            token: client token
+
+        Returns: removed Client object
+
+        """
+        with self.lock:
+            client = self.clients.pop(token)
+            self.logger.info(
+                "Client Name:{} \tToken: {} left.  Total clients: {}".format(client.name, token, len(self.clients))
+            )
+            return client
+
+    def login_client(self, client_login, context):
+        """
+        validate the client state message
+
+        :param context: gRPC connection context
+        :return: client id if it's a valid client
+        """
+        if not self.is_valid_task(client_login.meta.project):
+            context.abort(grpc.StatusCode.INVALID_ARGUMENT, "Requested task does not match the current server task")
+        return self.authenticated_client(client_login, context)
+
+    def validate_client(self, client_state, context, allow_new=False):
+        """
+        validate the client state message
+
+        :param client_state: A ClientState message received by server
+        :param context: gRPC connection context
+        :param allow_new: whether to allow new client. Its task should
+            still match server's.
+        :return: client id if it's a valid client
+        """
+
+        token = client_state.token
+        if not token:
+            context.abort(grpc.StatusCode.INVALID_ARGUMENT, "Could not read client uid from the payload")
+            client = None
+        elif not self.is_valid_task(client_state.meta.project):
+            context.abort(grpc.StatusCode.INVALID_ARGUMENT, "Requested task does not match the current server task")
+            client = None
+        elif not (allow_new or self.is_from_authorized_client(token)):
+            context.abort(grpc.StatusCode.UNAUTHENTICATED, "Unknown client identity")
+            client = None
+        else:
+            client = self.clients.get(token)
+        return client
+
+    def authenticated_client(self, client_login, context):
+        """
+        Use SSL certificate for authenticate the client.
+        :param client_login:
+        :param context:
+        :return:
+        """
+        client = self.clients.get(client_login.token)
+        if not client:
+            cn_names = context.auth_context().get("x509_common_name")
+            if cn_names:
+                client_name = cn_names[0].decode("utf-8")
+                if client_login.client_name:
+                    if not client_login.client_name == client_name:
+                        context.abort(
+                            grpc.StatusCode.UNAUTHENTICATED, "client ID does not match the SSL certificate CN"
+                        )
+                        return None
+            else:
+                client_name = client_login.client_name
+
+            for token, client in self.clients.items():
+                if client.name == client_name:
+                    context.abort(
+                        grpc.StatusCode.FAILED_PRECONDITION,
+                        "Client ID already registered as a client: {}".format(client_name),
+                    )
+                    return None
+
+            client = Client(client_name, str(uuid.uuid4()))
+        return client
+
+    def is_from_authorized_client(self, client_id):
+        """
+        simple authentication of the client.
+
+        :return: True indicates it is a recognised client
+        """
+        return client_id in self.clients
+
+    def is_valid_task(self, task):
+        """
+        check whether the requested task matches the server's task
+        """
+        return task.name == self.project_name
+
+    def heartbeat(self, token, client_id, context):
+        """
+        update the heartbeat of the client.
+        :param token: client ID token
+        :return: If a new client needs to be created.
+        """
+        with self.lock:
+            client = self.clients.get(token)
+            if client:
+                client.last_connect_time = time.time()
+                # self.clients.update({token: time.time()})
+                self.logger.debug(f"Receive heartbeat from Client:{token}")
+                return False
+            else:
+                for _token, _client in self.clients.items():
+                    if _client.name == client_id:
+                        context.abort(
+                            grpc.StatusCode.FAILED_PRECONDITION,
+                            "Client ID already registered as a client: {}".format(client_id),
+                        )
+                        self.logger.info(
+                            "Failed to re-activate dead client:{} with token: {}. Client already exist.".format(
+                                client_id, _token
+                            )
+                        )
+                        return False
+
+                client = Client(client_id, token)
+                client.last_connect_time = time.time()
+                # self._set_instance_name(client)
+                self.clients.update({token: client})
+                self.logger.info("Re-activate dead client:{} with token: {}".format(client_id, token))
+
+                return True
+        # return self.clients
+
+    def get_clients(self):
+        """
+        get the list of registered clients.
+        :return:
+        """
+        return self.clients
+
+    def get_min_clients(self):
+        return self.min_num_clients
+
+    def get_max_clients(self):
+        return self.max_num_clients
diff --git a/nvflare/private/fed/server/cmd_utils.py b/nvflare/private/fed/server/cmd_utils.py
new file mode 100644
index 0000000000..6b8ad3983c
--- /dev/null
+++ b/nvflare/private/fed/server/cmd_utils.py
@@ -0,0 +1,208 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from nvflare.apis.server_engine_spec import ServerEngineSpec
+from nvflare.fuel.hci.conn import Connection
+from nvflare.security.security import Action, FLAuthzContext
+
+
+class CommandUtil(object):
+
+    TARGET_CLIENTS = "target_clients"
+    TARGET_CLIENT_TOKENS = "target_client_tokens"
+    TARGET_CLIENT_NAMES = "target_client_names"
+    TARGET_TYPE = "target_type"
+
+    TARGET_TYPE_CLIENT = "client"
+    TARGET_TYPE_SERVER = "server"
+    TARGET_TYPE_ALL = "all"
+
+    SITE_SERVER = "server"
+
+    def validate_command_targets(self, conn: Connection, args: List[str]) -> str:
+        """
+        Validate specified args and determine and set target type and target names in conn.
+        The args must be like this:
+
+            target_type client_names ...
+
+        where target_type is one of 'all', 'client', 'server'
+
+        Args:
+            conn:
+            args:
+
+        Returns: error message if error found
+
+        """
+        # return target type and a list of target names
+        if len(args) < 1:
+            return "missing target type (server or client)"
+
+        target_type = args[0]
+        conn.set_prop(self.TARGET_TYPE, target_type)
+
+        if target_type == self.TARGET_TYPE_SERVER:
+            return ""
+
+        if target_type == self.TARGET_TYPE_CLIENT:
+            client_names = args[1:]
+        elif target_type == self.TARGET_TYPE_ALL:
+            client_names = []
+        else:
+            return "unknown target type {}".format(target_type)
+
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineSpec)
+        if len(client_names) == 0:
+            # get all clients
+            clients = engine.get_clients()
+        else:
+            clients, invalid_inputs = engine.validate_clients(client_names)
+            if invalid_inputs:
+                return "invalid client(s): {}".format(" ".join(invalid_inputs))
+
+        if target_type == self.TARGET_TYPE_CLIENT and not clients:
+            return "no clients available"
+
+        valid_tokens = []
+        client_names = []
+        all_clients = {}
+        for c in clients:
+            valid_tokens.append(c.token)
+            client_names.append(c.name)
+            all_clients[c.token] = c.name
+        conn.set_prop(self.TARGET_CLIENT_TOKENS, valid_tokens)
+        # if clients:
+        #     client_names = [c.name for c in clients]
+        # else:
+        #     client_names = []
+        conn.set_prop(self.TARGET_CLIENT_NAMES, client_names)
+        conn.set_prop(self.TARGET_CLIENTS, all_clients)
+        return ""
+
+    def _authorize_actions(self, conn: Connection, args: List[str], actions):
+        err = self.validate_command_targets(conn, args)
+        if err:
+            conn.append_error(err)
+            return False, None
+
+        target_type = conn.get_prop(self.TARGET_TYPE)
+        authorize_server = False
+        authorize_clients = False
+
+        if target_type == self.TARGET_TYPE_SERVER:
+            authorize_server = True
+        elif target_type == self.TARGET_TYPE_CLIENT:
+            authorize_clients = True
+        else:
+            # all
+            authorize_server = True
+            authorize_clients = True
+
+        sites = []
+        if authorize_clients:
+            client_names = conn.get_prop(self.TARGET_CLIENT_NAMES)
+
+            if client_names:
+                sites.extend(client_names)
+
+        if authorize_server:
+            sites.append(self.SITE_SERVER)
+
+        authz_ctx = FLAuthzContext.new_authz_context(site_names=sites, actions=actions)
+        return True, authz_ctx
+
+    def authorize_view(self, conn: Connection, args: List[str]):
+        return self._authorize_actions(conn, args[1:], [Action.VIEW])
+
+    def authorize_train(self, conn: Connection, args: List[str]):
+        return self._authorize_actions(conn, args[1:], [Action.TRAIN])
+
+    def authorize_operate(self, conn: Connection, args: List[str]):
+        return self._authorize_actions(conn, args[1:], [Action.OPERATE])
+
+    def send_request_to_clients(self, conn, message, process_client_replies=None):
+        client_tokens = conn.get_prop(self.TARGET_CLIENT_TOKENS)
+
+        # for client in clients:
+        #     requests.update({client.strip(): message})
+
+        # client_names = conn.get_prop(self.TARGET_CLIENT_NAMES, None)
+        if not client_tokens:
+            return None
+
+        requests = {}
+        for token in client_tokens:
+            requests.update({token: message})
+
+        admin_server = conn.server
+        replies = admin_server.send_requests(requests, timeout_secs=admin_server.timeout)
+
+        if process_client_replies:
+            return process_client_replies(replies)
+        else:
+            return replies
+
+    def process_replies_to_table(self, conn: Connection, replies):
+        """
+        Display the responses in a table format.
+        Args:
+            conn:
+            replies: replies from clients
+
+        Returns:
+
+        """
+        if not replies:
+            conn.append_string("no responses from clients")
+
+        engine = conn.app_ctx
+        table = conn.append_table(["Client", "Response"])
+        for r in replies:
+            if r.reply:
+                resp = r.reply.body
+            else:
+                resp = ""
+            client_name = engine.get_client_name_from_token(r.client_token)
+            if not client_name:
+                clients = conn.get_prop(self.TARGET_CLIENTS)
+                client_name = clients.get(r.client_token, "")
+
+            table.add_row([client_name, resp])
+
+    def process_replies_to_string(self, conn: Connection, replies) -> str:
+        """
+        Display thee responses in String format.
+        Args:
+            conn:
+            replies: replies from clients
+
+        Returns:
+
+        """
+        engine = conn.app_ctx
+        response = "no responses from clients"
+        if replies:
+            response = ""
+            for r in replies:
+                client_name = engine.get_client_name_from_token(r.client_token)
+                response += "client:" + client_name
+                if r.reply:
+                    response += " : " + r.reply.body + "\n"
+                else:
+                    response += " : No replies\n"
+        return response
diff --git a/nvflare/private/fed/server/comp_caller_cmd.py b/nvflare/private/fed/server/comp_caller_cmd.py
new file mode 100644
index 0000000000..7647701ea0
--- /dev/null
+++ b/nvflare/private/fed/server/comp_caller_cmd.py
@@ -0,0 +1,116 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec
+from nvflare.private.fed.server.server_engine_internal_spec import ServerEngineInternalSpec
+from nvflare.security.security import Action, FLAuthzContext
+from nvflare.widgets.comp_caller import ComponentCaller
+from nvflare.widgets.widget import WidgetID
+
+from .cmd_utils import CommandUtil
+
+
+class ComponentCallerCommandModule(CommandModule, CommandUtil):
+
+    _CONN_KEY_CALLER = "caller"
+    _CONN_KEY_TARGET = "comp_target"
+    _CONN_KEY_ACTION = "action"
+    _CONN_KEY_PARAMS = "params"
+
+    def get_spec(self):
+        return CommandModuleSpec(
+            name="admin",
+            cmd_specs=[
+                CommandSpec(
+                    name="call",
+                    description="issue a call to components",
+                    usage="call comp_target action params",
+                    handler_func=self.call_component,
+                    authz_func=self.authorize_call_component,
+                    visible=True,
+                )
+            ],
+        )
+
+    def authorize_call_component(self, conn: Connection, args: [str]):
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        caller = engine.get_widget(WidgetID.COMPONENT_CALLER)
+        if not caller:
+            conn.append_error("component caller not available")
+            return False, None
+
+        if not isinstance(caller, ComponentCaller):
+            conn.append_error("system error: component caller not right object")
+            return False, None
+
+        conn.set_prop(self._CONN_KEY_CALLER, caller)
+
+        run_info = engine.get_run_info()
+        if not run_info or run_info.run_number < 0:
+            conn.append_string("App is not running")
+            return False, None
+
+        # validate the command
+        if len(args) < 3:
+            conn.append_error("Syntax error. Usage: {} comp_target action [params...]".format(args[0]))
+            return False, None
+
+        comp_target = args[1]
+        action = args[2]
+        param_args = args[3:]
+
+        # parse params
+        params_dict = {}
+        if param_args:
+            for a in param_args:
+                parts = a.split()
+                if len(parts) <= 0:
+                    conn.append_error("missing params")
+                    return False, None
+
+                # each param part must be: key=value
+                for p in parts:
+                    kvs = p.split("=")
+                    if len(kvs) != 2:
+                        conn.append_error("Syntax error: params must be key/value pairs separated by =")
+                        return False, None
+                    params_dict[kvs[0]] = kvs[1]
+
+        conn.set_prop(self._CONN_KEY_ACTION, action)
+        conn.set_prop(self._CONN_KEY_TARGET, comp_target)
+        conn.set_prop(self._CONN_KEY_PARAMS, params_dict)
+
+        return True, FLAuthzContext.new_authz_context(site_names=["server"], actions=[Action.TRAIN])
+
+    def call_component(self, conn: Connection, args: [str]):
+        # only support server side for now
+        caller = conn.get_prop(self._CONN_KEY_CALLER)
+        assert isinstance(caller, ComponentCaller)
+        action = conn.get_prop(self._CONN_KEY_ACTION)
+        comp_target = conn.get_prop(self._CONN_KEY_TARGET)
+        call_params = conn.get_prop(self._CONN_KEY_PARAMS)
+
+        result = caller.call_components(target=comp_target, action=action, params=call_params)
+
+        if not result:
+            conn.append_string("No result: no component responded to the call")
+            return
+
+        # the result is a dict of: target => response
+        table = conn.append_table(["Component", "Response"])
+        for comp_name, resp in result.items():
+            table.add_row([comp_name, resp])
diff --git a/nvflare/private/fed/server/fed_server.py b/nvflare/private/fed/server/fed_server.py
new file mode 100644
index 0000000000..19ac780a0b
--- /dev/null
+++ b/nvflare/private/fed/server/fed_server.py
@@ -0,0 +1,678 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""federated server for aggregating and sharing federated model"""
+
+import logging
+import pickle
+import threading
+import time
+from abc import abstractmethod
+from concurrent import futures
+from threading import Lock
+from typing import List, Optional
+
+import grpc
+from google.protobuf.struct_pb2 import Struct
+from google.protobuf.timestamp_pb2 import Timestamp
+
+import nvflare.private.fed.protos.admin_pb2 as admin_msg
+import nvflare.private.fed.protos.admin_pb2_grpc as admin_service
+import nvflare.private.fed.protos.federated_pb2 as fed_msg
+import nvflare.private.fed.protos.federated_pb2_grpc as fed_service
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import MachineStatus, FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, ReservedHeaderKey
+from nvflare.apis.workspace import Workspace
+from nvflare.private.defs import SpecialTaskName
+from nvflare.private.fed.server.server_runner import ServerRunner
+from nvflare.private.fed.utils.messageproto import message_to_proto, proto_to_message
+from nvflare.private.fed.utils.numproto import proto_to_bytes
+from nvflare.widgets.fed_event import ServerFedEventRunner
+from .client_manager import ClientManager
+from .run_manager import RunManager
+from .server_engine import ServerEngine
+from .server_status import ServerStatus
+from ..utils.fed_utils import shareable_to_modeldata
+
+GRPC_DEFAULT_OPTIONS = [
+    ("grpc.max_send_message_length", 1024 * 1024 * 1024),
+    ("grpc.max_receive_message_length", 1024 * 1024 * 1024),
+]
+
+
+class BaseServer:
+    """
+    Base FL server, provides the clients management, server deployment.
+    """
+    def __init__(
+        self,
+        project_name=None,
+        min_num_clients=2,
+        max_num_clients=10,
+        heart_beat_timeout=600,
+        handlers: Optional[List[FLComponent]] = None,
+    ):
+        self.project_name = project_name  # should uniquely define a tlt workflow
+        self.min_num_clients = max(min_num_clients, 1)
+        self.max_num_clients = max(max_num_clients, 1)
+
+        self.heart_beat_timeout = heart_beat_timeout
+        self.handlers: [FLComponent] = handlers
+        # self.cmd_modules = cmd_modules
+
+        self.client_manager = ClientManager(
+            project_name=self.project_name, min_num_clients=self.min_num_clients, max_num_clients=self.max_num_clients
+        )
+
+        self.grpc_server = None
+        self.admin_server = None
+        self.lock = Lock()
+        self.fl_ctx = FLContext()
+        self.platform = None
+
+        self.shutdown = False
+        self.status = ServerStatus.NOT_STARTED
+
+        self.abort_signal = None
+
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def get_all_clients(self):
+        return self.client_manager.get_clients()
+
+    @abstractmethod
+    def remove_client_data(self, token):
+        pass
+
+    def close(self):
+        """
+        shutdown the server.
+
+        :return:
+        """
+        try:
+            if self.lock:
+                self.lock.release()
+        except RuntimeError:
+            self.logger.info("canceling sync locks")
+        try:
+            if self.grpc_server:
+                self.grpc_server.stop(0)
+            # if self.handlers:
+            #     for h in self.handlers:
+            #         h.shutdown(self.fl_ctx)
+            # if self.admin_server:
+            #     self.admin_server.stop()
+        finally:
+            self.logger.info("server off")
+            return 0
+
+    def deploy(self, grpc_args=None, secure_train=False):
+        """
+        start a grpc server and listening the designated port.
+        :param grpc_args:
+        """
+        num_server_workers = grpc_args.get("num_server_workers", 1)
+        num_server_workers = max(self.client_manager.get_min_clients(), num_server_workers)
+        target = grpc_args["service"].get("target", "0.0.0.0:6007")
+        grpc_options = grpc_args["service"].get("options", GRPC_DEFAULT_OPTIONS)
+
+        compression = grpc.Compression.NoCompression
+        if "Deflate" == grpc_args.get("compression"):
+            compression = grpc.Compression.Deflate
+        elif "Gzip" == grpc_args.get("compression"):
+            compression = grpc.Compression.Gzip
+
+        if not self.grpc_server:
+            self.grpc_server = grpc.server(
+                futures.ThreadPoolExecutor(max_workers=num_server_workers),
+                options=grpc_options,
+                compression=compression,
+            )
+            fed_service.add_FederatedTrainingServicer_to_server(self, self.grpc_server)
+            admin_service.add_AdminCommunicatingServicer_to_server(self, self.grpc_server)
+
+        if secure_train:
+            with open(grpc_args["ssl_private_key"], "rb") as f:
+                private_key = f.read()
+            with open(grpc_args["ssl_cert"], "rb") as f:
+                certificate_chain = f.read()
+            with open(grpc_args["ssl_root_cert"], "rb") as f:
+                root_ca = f.read()
+
+            server_credentials = grpc.ssl_server_credentials(
+                (
+                    (
+                        private_key,
+                        certificate_chain,
+                    ),
+                ),
+                root_certificates=root_ca,
+                require_client_auth=True,
+            )
+            self.grpc_server.add_secure_port(target, server_credentials)
+            self.logger.info("starting secure server at %s", target)
+        else:
+            self.grpc_server.add_insecure_port(target)
+            self.logger.info("starting insecure server at %s", target)
+        self.grpc_server.start()
+
+        # return self.start()
+        cleanup_thread = threading.Thread(target=self.client_cleanup)
+        # heartbeat_thread.daemon = True
+        cleanup_thread.start()
+
+    def client_cleanup(self):
+        while not self.shutdown:
+            self.remove_dead_clients()
+            time.sleep(15)
+
+    def set_admin_server(self, admin_server):
+        self.admin_server = admin_server
+
+    def remove_dead_clients(self):
+        # Clean and remove the dead client without heartbeat.
+        self.logger.debug("trying to remove dead clients .......")
+        delete = []
+        for token, client in self.client_manager.get_clients().items():
+            if client.last_connect_time < time.time() - self.heart_beat_timeout:
+                delete.append(token)
+        for token in delete:
+            client = self.client_manager.remove_client(token)
+            # self.tokens.pop(token, None)
+            self.remove_client_data(token)
+            if self.admin_server:
+                self.admin_server.client_dead(client.name)
+            self.logger.info(
+                "Remove the dead Client. Name: {}\t Token: {}.  Total clients: {}".format(
+                    client.name, token, len(self.client_manager.get_clients())
+                )
+            )
+
+    def fl_shutdown(self):
+        self.shutdown = True
+        self.close()
+
+
+class FederatedServer(BaseServer, fed_service.FederatedTrainingServicer, admin_service.AdminCommunicatingServicer):
+    """
+    Federated server services
+    """
+
+    def __init__(
+        self,
+        project_name=None,
+        min_num_clients=2,
+        max_num_clients=10,
+        wait_after_min_clients=10,
+        cmd_modules=None,
+        heart_beat_timeout=600,
+        handlers: Optional[List[FLComponent]] = None,
+        args=None,
+        secure_train=False,
+        enable_byoc=False
+    ):
+        """
+
+        :param project_name: server project name.
+        :param min_num_clients: minimum number of contributors at each round.
+        :param max_num_clients: maximum number of contributors at each round.
+        """
+        # assert model_log_dir is not None, "model_log_dir must be specified to save checkpoint"
+
+        self.logger = logging.getLogger("FederatedServer")
+
+        BaseServer.__init__(
+            self,
+            project_name=project_name,
+            min_num_clients=min_num_clients,
+            max_num_clients=max_num_clients,
+            heart_beat_timeout=heart_beat_timeout,
+            handlers=handlers,
+        )
+
+        self.contributed_clients = {}
+        self.tokens = None
+        self.round_started = Timestamp()
+
+        with self.lock:
+            self.reset_tokens()
+
+        self.wait_after_min_clients = wait_after_min_clients
+
+        self.cmd_modules = cmd_modules
+
+        self.builder = None
+
+        # Additional fields for CurrentTask meta_data in GetModel API.
+        self.current_model_meta_data = {}
+
+        self.engine = ServerEngine(self, args, self.client_manager)
+        self.run_manager = None
+        self.server_runner = None
+
+        self.processors = {}
+        self.runner_config = None
+        self.secure_train = secure_train
+        self.enable_byoc = enable_byoc
+
+    def get_current_model_meta_data(self):
+        """Get the model meta data, which usually contains additional fields"""
+        s = Struct()
+        for k, v in self.current_model_meta_data.items():
+            s.update({k: v})
+        return s
+
+    @property
+    def task_meta_info(self):
+        """
+        the model_meta_info uniquely defines the current model,
+        it is used to reject outdated client's update.
+
+        :return: model meta data object
+        """
+        meta_info = fed_msg.MetaData()
+        meta_info.created.CopyFrom(self.round_started)
+        meta_info.project.name = self.project_name
+        return meta_info
+
+    # def register_processor(self, processor: ResultProcessor):
+    #     # assert isinstance(processor, ResultProcessor), 'processor must be ResultProcessor'
+    #
+    #     topics = processor.get_topics()
+    #     for topic in topics:
+    #         assert topic not in self.processors, "duplicate processors for topic {}".format(topic)
+    #         self.processors[topic] = processor
+    #
+    def remove_client_data(self, token):
+        self.tokens.pop(token, None)
+
+    def reset_tokens(self):
+        """
+        restart the token set, so that each client can take a token
+        and start fetching the current global model.
+        This function is not thread-safe.
+        """
+        # last_time = self.round_started.seconds
+        # self.round_started.GetCurrentTime()
+        # now_time = self.round_started.seconds
+        # reset_duration = now_time - last_time
+        # self.logger.info("Round time: %s second(s).", reset_duration if reset_duration > 0 else "less than a")
+
+        self.tokens = dict()
+        for client in self.get_all_clients().keys():
+            self.tokens[client] = self.task_meta_info
+
+    def Register(self, request, context):
+        """
+        register new clients on the fly.
+        Each client must get registered before getting the global model.
+        The server will expect updates from the registered clients
+        for multiple federated rounds.
+
+        This function does not change min_num_clients and max_num_clients.
+        """
+        # fire_event(EventType.CLIENT_REGISTER, self.handlers, self.fl_ctx)
+
+        token = self.client_manager.authenticate(request, context)
+        if token:
+            self.tokens[token] = self.task_meta_info
+            if self.admin_server:
+                self.admin_server.client_heartbeat(token)
+
+            return fed_msg.FederatedSummary(comment="New client registered", token=token)
+
+    def Quit(self, request, context):
+        """
+        existing client quits the federated training process.
+        Server will stop sharing the global model with the client,
+        further contribution will be rejected.
+
+        This function does not change min_num_clients and max_num_clients.
+        """
+        # fire_event(EventType.CLIENT_QUIT, self.handlers, self.fl_ctx)
+
+        client = self.client_manager.validate_client(request, context)
+        if client:
+            token = client.get_token()
+
+            client = self.client_manager.remove_client(token)
+            self.tokens.pop(token, None)
+            if self.admin_server:
+                self.admin_server.client_dead(token)
+
+        return fed_msg.FederatedSummary(comment="Removed client")
+
+    def GetTask(self, request, context):
+        """
+        process client's request of the current global model
+        """
+        # # fl_ctx = self.fl_ctx.clone_sticky()
+        # if not self.run_manager:
+        #     context.abort(grpc.StatusCode.OUT_OF_RANGE, "Server training stopped")
+
+        # if self.server_runner is None:
+        #     context.abort(grpc.StatusCode.OUT_OF_RANGE, "Server has stopped")
+
+        with self.engine.new_context() as fl_ctx:
+
+            client = self.client_manager.validate_client(request, context)
+            if client is None:
+                context.abort(grpc.StatusCode.FAILED_PRECONDITION, "Client not valid.")
+
+            self.logger.info(f"Fetch task requested from client: {client.name} ({client.get_token()})")
+            token = client.get_token()
+
+            engine = fl_ctx.get_engine()
+            # shared_fl_ctx = FLContext()
+            # shared_fl_ctx.set_run_number(request.meta.run_number)
+            shared_fl_ctx = pickle.loads(proto_to_bytes(request.context["fl_context"]))
+            fl_ctx.set_peer_context(shared_fl_ctx)
+
+            with self.lock:
+                # shareable = self.model_manager.get_shareable(self.fl_ctx)
+
+                if self.server_runner is None or engine is None or self.engine.run_manager is None:
+                    self.logger.info("server has no current run - asked client to end the run")
+                    taskname = SpecialTaskName.END_RUN
+                    task_id = ""
+                    shareable = None
+                else:
+                    # taskname, task_id, shareable = self.controller.process_task_request(client, fl_ctx)
+                    taskname, task_id, shareable = self.server_runner.process_task_request(client, fl_ctx)
+
+                if shareable is None:
+                    shareable = Shareable()
+
+                task = fed_msg.CurrentTask(task_name=taskname)
+                task.meta.CopyFrom(self.task_meta_info)
+                meta_data = self.get_current_model_meta_data()
+
+                # we need TASK_ID back as a cookie
+                shareable.add_cookie(name=FLContextKey.TASK_ID, data=task_id)
+
+                # we also need to make TASK_ID available to the client
+                shareable.set_header(key=FLContextKey.TASK_ID, value=task_id)
+
+                task.meta_data.CopyFrom(meta_data)
+
+                current_model = shareable_to_modeldata(shareable, fl_ctx)
+                task.data.CopyFrom(current_model)
+                self.logger.info(f"Return task:{taskname} to client:{client.name} --- ({token}) ")
+
+                # self.fl_ctx.merge_sticky(fl_ctx)
+
+                return task
+
+    def SubmitUpdate(self, request, context):
+        """
+        handling client's submission of the federated updates
+        running aggregation if there are enough updates
+        """
+        # if not self.run_manager:
+        #     context.abort(grpc.StatusCode.OUT_OF_RANGE, "Server has stopped")
+
+        if self.server_runner is None or self.engine.run_manager is None:
+            # context.abort(grpc.StatusCode.OUT_OF_RANGE, "Server has stopped")
+            self.logger.info("ignored result submission since Server Engine isn't ready")
+            context.abort(grpc.StatusCode.OUT_OF_RANGE, "Server has stopped")
+
+        # fl_ctx = self.fl_ctx.clone_sticky()
+        with self.engine.new_context() as fl_ctx:
+
+            # if self.status == ServerStatus.TRAINING_STOPPED or self.status == ServerStatus.TRAINING_NOT_STARTED:
+            #     context.abort(grpc.StatusCode.OUT_OF_RANGE, "Server training stopped")
+            #     return
+
+            contribution = request
+
+            client = self.client_manager.validate_client(contribution.client, context)
+            if client is None:
+                response_comment = "Ignored the submit from invalid client. "
+                self.logger.info(response_comment)
+            else:
+                token = client.get_token()
+
+                with self.lock:
+                    shareable = Shareable()
+                    shareable = shareable.from_bytes(proto_to_bytes(request.data.params["data"]))
+                    shared_fl_context = pickle.loads(proto_to_bytes(request.data.params["fl_context"]))
+
+                    # fl_ctx.set_prop(FLContextKey.PEER_CONTEXT, shared_fl_context)
+                    fl_ctx.set_peer_context(shared_fl_context)
+
+                    shared_fl_context.set_prop(FLContextKey.SHAREABLE, shareable, private=False)
+
+                    contribution_meta = contribution.client.meta
+                    client_contrib_id = "{}_{}_{}".format(
+                        contribution_meta.project.name, client.name, contribution_meta.current_round
+                    )
+                    contribution_task_name = contribution.task_name
+
+                    timenow = Timestamp()
+                    timenow.GetCurrentTime()
+                    time_seconds = timenow.seconds - self.round_started.seconds
+                    self.logger.info(
+                        "received update from %s (%s Bytes, %s seconds)",
+                        client_contrib_id,
+                        contribution.ByteSize(),
+                        time_seconds or "less than 1",
+                    )
+
+                    # fire_event(EventType.BEFORE_PROCESS_SUBMISSION, self.handlers, fl_ctx)
+
+                    if self.save_contribution(client_contrib_id, contribution):
+                        assert isinstance(shareable, Shareable)
+                        # task_id = shared_fl_context.get_cookie(FLContextKey.TASK_ID)
+                        task_id = shareable.get_cookie(FLContextKey.TASK_ID)
+                        self.server_runner.process_submission(
+                            client, contribution_task_name, task_id, shareable, fl_ctx
+                        )
+
+                        # fire_event(EventType.AFTER_PROCESS_SUBMISSION, self.handlers, fl_ctx)
+
+            response_comment = "Received from {} ({} Bytes, {} seconds)".format(
+                contribution.client.client_name,
+                contribution.ByteSize(),
+                time_seconds or "less than 1",
+            )
+            summary_info = fed_msg.FederatedSummary(comment=response_comment)
+            summary_info.meta.CopyFrom(self.task_meta_info)
+
+            # with self.lock:
+            #     self.fl_ctx.merge_sticky(fl_ctx)
+
+            return summary_info
+
+    def AuxCommunicate(self, request, context):
+        """
+        handling client's submission of the federated updates
+        running aggregation if there are enough updates
+        """
+        # if not self.run_manager:
+        #     context.abort(grpc.StatusCode.OUT_OF_RANGE, "Server has stopped")
+
+        if self.server_runner is None or self.engine.run_manager is None:
+            # context.abort(grpc.StatusCode.OUT_OF_RANGE, "Server has stopped")
+            self.logger.info("ignored result submission since Server Engine isn't ready")
+            context.abort(grpc.StatusCode.OUT_OF_RANGE, "Server has stopped")
+
+        # fl_ctx = self.fl_ctx.clone_sticky()
+        with self.engine.new_context() as fl_ctx:
+
+            contribution = request
+
+            client = self.client_manager.validate_client(contribution.client, context)
+            if client is None:
+                response_comment = "Ignored the submit from invalid client. "
+                self.logger.info(response_comment)
+            else:
+                token = client.get_token()
+
+            shareable = Shareable()
+            shareable = shareable.from_bytes(proto_to_bytes(request.data["data"]))
+            shared_fl_context = pickle.loads(proto_to_bytes(request.data["fl_context"]))
+
+            fl_ctx.set_peer_context(shared_fl_context)
+            shareable.set_peer_props(shared_fl_context.get_all_public_props())
+
+            shared_fl_context.set_prop(FLContextKey.SHAREABLE, shareable, private=False)
+
+            topic = shareable.get_header(ReservedHeaderKey.TOPIC)
+            # aux_runner = fl_ctx.get_aux_runner()
+            # assert isinstance(aux_runner, ServerAuxRunner)
+            reply = self.engine.dispatch(topic=topic, request=shareable, fl_ctx=fl_ctx)
+
+            aux_reply = fed_msg.AuxReply()
+            aux_reply.data.CopyFrom(shareable_to_modeldata(reply, fl_ctx))
+
+            return aux_reply
+
+    def Heartbeat(self, request, context):
+        token = request.token
+        cn_names = context.auth_context().get("x509_common_name")
+        if cn_names:
+            client_name = cn_names[0].decode("utf-8")
+        else:
+            client_name = request.client_name
+
+        if self.client_manager.heartbeat(token, client_name, context):
+            self.tokens[token] = self.task_meta_info
+        if self.admin_server:
+            self.admin_server.client_heartbeat(token)
+
+        summary_info = fed_msg.FederatedSummary()
+        return summary_info
+
+    def Retrieve(self, request, context):
+        client_name = request.client_name
+        messages = self.admin_server.get_outgoing_requests(client_token=client_name) if self.admin_server else []
+
+        response = admin_msg.Messages()
+        # response.message.CopyFrom(messages)
+        for m in messages:
+            # message = response.message.add()
+            response.message.append(message_to_proto(m))
+        return response
+
+    def SendReply(self, request, context):
+        client_name = request.client_name
+        message = proto_to_message(request.message)
+        if self.admin_server:
+            self.admin_server.accept_reply(client_token=client_name, reply=message)
+
+        response = admin_msg.Empty()
+        return response
+
+    def SendResult(self, request, context):
+        client_name = request.client_name
+        message = proto_to_message(request.message)
+
+        processor = self.processors.get(message.topic)
+        processor.process(client_name, message)
+
+        response = admin_msg.Empty()
+        return response
+
+    def save_contribution(self, client_contrib_id, data):
+        """
+        save the client's current contribution.
+
+        :return: True iff it is successfully saved
+        """
+        return True
+
+    def start_run(self, run_number, run_root, conf, args):
+        # self.status = ServerStatus.STARTING
+
+        # Create the FL Engine
+        workspace = Workspace(args.workspace, "server", args.config_folder)
+        self.run_manager = RunManager(
+            server_name=self.project_name,
+            engine=self.engine,
+            run_num=run_number,
+            workspace=workspace,
+            components=self.runner_config.components,
+            client_manager=self.client_manager,
+            handlers=self.runner_config.handlers,
+        )
+        self.engine.set_run_manager(self.run_manager)
+        self.engine.set_configurator(conf)
+        self.engine.asked_to_stop = False
+
+        fed_event_runner = ServerFedEventRunner()
+        self.run_manager.add_handler(fed_event_runner)
+
+        try:
+            with self.engine.new_context() as fl_ctx:
+
+                # with open(os.path.join(run_root, env_config)) as file:
+                #     env = json.load(file)
+
+                fl_ctx.set_prop(FLContextKey.APP_ROOT, run_root, sticky=True)
+                fl_ctx.set_prop(FLContextKey.CURRENT_RUN, run_number, private=False, sticky=True)
+                fl_ctx.set_prop(FLContextKey.WORKSPACE_ROOT, args.workspace, private=True, sticky=True)
+                fl_ctx.set_prop(FLContextKey.ARGS, args, private=True, sticky=True)
+                fl_ctx.set_prop(FLContextKey.WORKSPACE_OBJECT, workspace, private=True)
+                fl_ctx.set_prop(FLContextKey.SECURE_MODE, self.secure_train, private=True, sticky=True)
+
+            self.server_runner = ServerRunner(config=self.runner_config, run_num=run_number, engine=self.engine)
+            self.run_manager.add_handler(self.server_runner)
+
+            # self.controller.initialize_run(self.fl_ctx)
+
+            # return super().start()
+            # self.status = ServerStatus.STARTED
+            # self.run_engine()
+            engine_thread = threading.Thread(target=self.run_engine)
+            # heartbeat_thread.daemon = True
+            engine_thread.start()
+
+            while self.engine.engine_info.status != MachineStatus.STOPPED:
+                # self.remove_dead_clients()
+                if self.engine.asked_to_stop:
+                    self.engine.abort_app_on_server()
+
+                time.sleep(3)
+
+        finally:
+            self.engine.engine_info.status = MachineStatus.STOPPED
+            self.engine.run_manager = None
+            self.run_manager = None
+
+    def abort_run(self):
+        with self.engine.new_context() as fl_ctx:
+            if self.server_runner:
+                self.server_runner.abort(fl_ctx)
+
+    def run_engine(self):
+        self.engine.engine_info.status = MachineStatus.STARTED
+        self.server_runner.run()
+        self.engine.engine_info.status = MachineStatus.STOPPED
+
+    def stop_training(self):
+        self.status = ServerStatus.STOPPED
+        self.logger.info("Server app stopped.\n\n")
+
+    def close(self):
+        """
+        shutdown the server.
+
+        :return:
+        """
+        self.logger.info("shutting down server")
+
+        return super().close()
diff --git a/nvflare/private/fed/server/info_coll_cmd.py b/nvflare/private/fed/server/info_coll_cmd.py
new file mode 100644
index 0000000000..6a6cbf3a9d
--- /dev/null
+++ b/nvflare/private/fed/server/info_coll_cmd.py
@@ -0,0 +1,144 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from typing import List
+
+from nvflare.apis.fl_constant import AdminCommandNames
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec
+from nvflare.private.defs import InfoCollectorTopic
+from nvflare.private.fed.server.admin import new_message
+from nvflare.private.fed.server.server_engine_internal_spec import ServerEngineInternalSpec
+from nvflare.widgets.info_collector import InfoCollector
+from nvflare.widgets.widget import WidgetID
+from .cmd_utils import CommandUtil
+
+
+class InfoCollectorCommandModule(CommandModule, CommandUtil):
+    """
+    NOTE: we only support Server side info collector commands for now,
+    due to the complexity of client-side process/child-process architecture.
+    """
+
+    CONN_KEY_COLLECTOR = "collector"
+
+    def get_spec(self):
+        return CommandModuleSpec(
+            name="info",
+            cmd_specs=[
+                CommandSpec(
+                    name=AdminCommandNames.SHOW_STATS,
+                    description="show current system stats",
+                    usage="show_stats",
+                    handler_func=self.show_stats,
+                    authz_func=self.authorize_info_collection,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.SHOW_ERRORS,
+                    description="show latest errors",
+                    usage="show_errors",
+                    handler_func=self.show_errors,
+                    authz_func=self.authorize_info_collection,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.RESET_ERRORS,
+                    description="reset errors",
+                    usage="reset_errors",
+                    handler_func=self.reset_errors,
+                    authz_func=self.authorize_info_collection,
+                    visible=True,
+                ),
+            ],
+        )
+
+    def authorize_info_collection(self, conn: Connection, args: List[str]):
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        collector = engine.get_widget(WidgetID.INFO_COLLECTOR)
+        if not collector:
+            conn.append_error("info collector not available")
+            return False, None
+
+        if not isinstance(collector, InfoCollector):
+            conn.append_error("system error: info collector not right object")
+            return False, None
+
+        conn.set_prop(self.CONN_KEY_COLLECTOR, collector)
+
+        run_info = engine.get_run_info()
+        if not run_info or run_info.run_number < 0:
+            conn.append_string("App is not running")
+            return False, None
+
+        # return True, FLAuthzContext.new_authz_context(
+        #     site_names=['server'],
+        #     actions=[Action.VIEW])
+        return self.authorize_view(conn, args)
+
+    def show_stats(self, conn: Connection, args: List[str]):
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        target_type = args[1]
+        if target_type == self.TARGET_TYPE_SERVER:
+            collector = conn.get_prop(self.CONN_KEY_COLLECTOR)
+            result = collector.get_run_stats()
+            conn.append_any(result)
+        elif target_type == self.TARGET_TYPE_CLIENT:
+            message = new_message(conn, topic=InfoCollectorTopic.SHOW_STATS, body="")
+            replies = self.send_request_to_clients(conn, message)
+            self._process_stats_replies(conn, replies)
+
+        # collector = conn.get_prop(self.CONN_KEY_COLLECTOR)
+        # result = collector.get_run_stats()
+        # conn.append_any(result)
+
+    def show_errors(self, conn: Connection, args: List[str]):
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        target_type = args[1]
+        if target_type == self.TARGET_TYPE_SERVER:
+            collector = conn.get_prop(self.CONN_KEY_COLLECTOR)
+            result = collector.get_errors()
+            conn.append_any(result)
+        elif target_type == self.TARGET_TYPE_CLIENT:
+            message = new_message(conn, topic=InfoCollectorTopic.SHOW_ERRORS, body="")
+            replies = self.send_request_to_clients(conn, message)
+            self._process_stats_replies(conn, replies)
+
+    def reset_errors(self, conn: Connection, args: List[str]):
+        collector = conn.get_prop(self.CONN_KEY_COLLECTOR)
+        collector.reset_errors()
+        conn.append_string("errors reset")
+
+    def _process_stats_replies(self, conn, replies):
+        if not replies:
+            conn.append_error("no responses from clients")
+            return
+
+        engine = conn.app_ctx
+        for r in replies:
+            client_name = engine.get_client_name_from_token(r.client_token)
+
+            conn.append_string(f"--- Client ---: {client_name}")
+            try:
+                body = json.loads(r.reply.body)
+                conn.append_any(body)
+            except BaseException:
+                conn.append_string("Bad responses from clients")
diff --git a/nvflare/private/fed/server/result_processor.py b/nvflare/private/fed/server/result_processor.py
new file mode 100644
index 0000000000..1ddd33a211
--- /dev/null
+++ b/nvflare/private/fed/server/result_processor.py
@@ -0,0 +1,50 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.private.admin_defs import Message
+
+
+class ResultProcessor(object):
+    """
+    The RequestProcessor is responsible for processing a request.
+    """
+
+    def get_topics(self) -> [str]:
+        """
+        Get topics that this processor will handle
+        :return: list of topics
+        """
+        pass
+
+    def process(self, client_name, req: Message):
+        """
+        Called to process the specified request
+        :param req:
+        :param app_ctx:
+        :return: a reply message
+        """
+        pass
+
+
+class ValidateResultProcessor(ResultProcessor):
+    def get_topics(self) -> [str]:
+        return ["validate"]
+
+    def process(self, client_name, message: Message):
+
+        print(
+            "Got the client: {} processor result. topic: {} \tbody: {}".format(
+                client_name, message.topic, str(message.body)
+            )
+        )
diff --git a/nvflare/private/fed/server/run_manager.py b/nvflare/private/fed/server/run_manager.py
new file mode 100644
index 0000000000..5652fc3229
--- /dev/null
+++ b/nvflare/private/fed/server/run_manager.py
@@ -0,0 +1,86 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import MachineStatus
+from nvflare.apis.fl_context import FLContext, FLContextManager
+from nvflare.apis.server_engine_spec import ServerEngineSpec
+from nvflare.apis.workspace import Workspace
+from nvflare.private.event import fire_event
+from .client_manager import ClientManager
+from .server_aux_runner import ServerAuxRunner
+
+
+class RunInfo(object):
+    def __init__(self, run_num, app_path):
+        self.run_number = run_num
+        self.start_time = time.time()
+        self.app_path = app_path
+        self.status = MachineStatus.STOPPED
+
+
+class RunManager:
+    def __init__(
+        self,
+        server_name,
+        engine: ServerEngineSpec,
+        run_num,
+        workspace: Workspace,
+        components: {str: FLComponent},
+        client_manager: [ClientManager] = None,
+        handlers: [FLComponent] = None,
+    ) -> None:
+        super().__init__()
+        self.server_name = server_name  # should uniquely define a tlt workflow
+
+        self.client_manager = client_manager
+        self.handlers = handlers
+        self.aux_runner = ServerAuxRunner()
+        self.add_handler(self.aux_runner)
+
+        self.fl_ctx_mgr = FLContextManager(
+            engine=engine, identity_name=server_name, run_num=run_num,
+            public_stickers={}, private_stickers={}
+        )
+
+        self.workspace = workspace
+        self.run_info = RunInfo(run_num=run_num, app_path=self.workspace.get_app_dir(run_num))
+
+        self.components = components
+
+    def get_server_name(self):
+        return self.server_name
+
+    def get_run_info(self):
+        return self.run_info
+
+    def get_handlers(self):
+        return self.handlers
+
+    def new_context(self) -> FLContext:
+        return self.fl_ctx_mgr.new_context()
+
+    def get_workspace(self) -> Workspace:
+        return self.workspace
+
+    def get_component(self, component_id: str) -> object:
+        return self.components.get(component_id)
+
+    def fire_event(self, event_type: str, fl_ctx: FLContext):
+        fire_event(event=event_type, handlers=self.handlers, ctx=fl_ctx)
+
+    def add_handler(self, handler: FLComponent):
+        self.handlers.append(handler)
diff --git a/nvflare/private/fed/server/server_aux_runner.py b/nvflare/private/fed/server/server_aux_runner.py
new file mode 100644
index 0000000000..dbd96975b3
--- /dev/null
+++ b/nvflare/private/fed/server/server_aux_runner.py
@@ -0,0 +1,117 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.client import Client
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, ReservedHeaderKey
+from nvflare.private.aux_runner import AuxRunner
+
+
+class ServerAuxRunner(AuxRunner):
+
+    """
+    Note: The ServerEngine must create a new ServerAuxRunner object for each RUN, and make sure
+    it is added as an event handler!
+
+    """
+
+    def __init__(self):
+        AuxRunner.__init__(self)
+
+    def send_aux_request(
+            self,
+            targets: [],
+            topic: str,
+            request: Shareable,
+            timeout: float,
+            fl_ctx: FLContext) -> dict:
+        """
+
+        Args:
+            targets: list of client names that the request will be sent to
+            topic: topic of the request
+            request: request data
+            timeout: how long to wait for result. 0 means fire-and-forget
+            fl_ctx: the FL context
+
+        Returns: a dict of results
+
+        """
+        if not isinstance(request, Shareable):
+            raise ValueError('invalid request type: expect Shareable but got {}'.format(type(request)))
+
+        if not targets:
+            raise ValueError('targets must be specified')
+
+        if targets is not None and not isinstance(targets, list):
+            raise TypeError('targets must be a list of Client or str, but got {}'.format(
+                type(targets)
+            ))
+
+        if not isinstance(topic, str):
+            raise TypeError('invalid topic: expects str but got {}'.format(
+                type(topic)))
+
+        if not topic:
+            raise ValueError('invalid topic: must not be empty')
+
+        if topic == self.TOPIC_BULK:
+            raise ValueError('topic value "{}" is reserved'.format(topic))
+
+        if not isinstance(timeout, float):
+            raise TypeError('invalid timeout: expects float but got {}'.format(
+                type(timeout)))
+
+        if timeout < 0:
+            raise ValueError('invalid timeout value {}: must >= 0.0'.format(timeout))
+
+        if not isinstance(fl_ctx, FLContext):
+            raise TypeError('invalid fl_ctx: expects FLContext but got {}'.format(
+                type(fl_ctx)))
+
+        request.set_peer_props(fl_ctx.get_all_public_props())
+        request.set_header(ReservedHeaderKey.TOPIC, topic)
+
+        engine = fl_ctx.get_engine()
+        # assert isinstance(engine, ServerEngineInternalSpec)
+
+        target_names = []
+        for t in targets:
+            if isinstance(t, str):
+                name = t
+            elif isinstance(t, Client):
+                name = t.name
+            else:
+                raise ValueError('invalid target in list: got {}'.format(type(t)))
+
+            if name not in target_names:
+                target_names.append(t)
+
+        clients, invalid_names = engine.validate_clients(target_names)
+        if invalid_names:
+            raise ValueError('invalid target(s): {}'.format(invalid_names))
+        valid_tokens = []
+        for c in clients:
+            if c.token not in valid_tokens:
+                valid_tokens.append(c.token)
+
+        replies = engine.aux_send(
+            targets=valid_tokens,
+            topic=topic,
+            request=request,
+            timeout=timeout,
+            fl_ctx=fl_ctx
+        )
+
+        return replies
diff --git a/nvflare/private/fed/server/server_cmd_modules.py b/nvflare/private/fed/server/server_cmd_modules.py
new file mode 100644
index 0000000000..766d36bc9b
--- /dev/null
+++ b/nvflare/private/fed/server/server_cmd_modules.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.fuel.hci.reg import CommandModule
+from nvflare.private.fed.server.info_coll_cmd import InfoCollectorCommandModule
+from nvflare.private.fed.server.shell_cmd import ShellCommandModule
+from nvflare.private.fed.server.sys_cmd import SystemCommandModule
+from nvflare.private.fed.server.training_cmds import TrainingCommandModule
+
+
+class ServerCommandModules:
+    cmd_modules = [ShellCommandModule(),
+                   SystemCommandModule(),
+                   TrainingCommandModule(),
+                   InfoCollectorCommandModule()]
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    @staticmethod
+    def register_cmd_module(cmd_module: CommandModule):
+        ServerCommandModules.cmd_modules.append(cmd_module)
diff --git a/nvflare/private/fed/server/server_engine.py b/nvflare/private/fed/server/server_engine.py
new file mode 100644
index 0000000000..0f457d15f5
--- /dev/null
+++ b/nvflare/private/fed/server/server_engine.py
@@ -0,0 +1,489 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gc
+import logging
+import os
+import pickle
+import re
+import shutil
+import sys
+import time
+import traceback
+from concurrent.futures import ThreadPoolExecutor
+from threading import Lock
+from typing import List, Tuple
+
+from nvflare.apis.client import Client
+from nvflare.apis.fl_constant import MachineStatus, ReservedTopic, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.apis.workspace import Workspace
+from nvflare.fuel.hci.zip_utils import zip_directory_to_bytes
+from nvflare.private.admin_defs import Message
+from nvflare.private.fed.server.server_json_config import ServerJsonConfigurator
+from nvflare.widgets.info_collector import InfoCollector
+from nvflare.widgets.widget import Widget, WidgetID
+
+from .run_manager import RunManager
+from .server_engine_internal_spec import EngineInfo, RunInfo, ServerEngineInternalSpec
+from .server_status import ServerStatus
+
+
+class ServerEngine(ServerEngineInternalSpec):
+    def __init__(self, server, args, client_manager, workers=3):
+        self.server = server
+        self.args = args
+        self.run_number = -1
+        self.run_manager = None
+        self.conf = None
+        self.client_manager = client_manager
+
+        self.widgets = {
+            WidgetID.INFO_COLLECTOR: InfoCollector(),
+            # WidgetID.FED_EVENT_RUNNER: ServerFedEventRunner()
+        }
+
+        self.engine_info = EngineInfo()
+
+        assert workers >= 1, "workers must >= 1"
+        self.executor = ThreadPoolExecutor(max_workers=workers)
+        self.lock = Lock()
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+        self.asked_to_stop = False
+
+    def _get_server_app_folder(self):
+        return "app_server"
+
+    def _get_client_app_folder(self, client_name):
+        return "app_" + client_name
+
+    def _get_run_folder(self):
+        return os.path.join(self.args.workspace, "run_" + str(self.run_number))
+
+    def get_engine_info(self) -> EngineInfo:
+        if self.run_number > 0:
+            run_folder = os.path.join(self.args.workspace, "run_" + str(self.run_number))
+            app_file = os.path.join(run_folder, "fl_app.txt")
+            if os.path.exists(app_file):
+                with open(app_file, "r") as f:
+                    self.engine_info.app_name = f.readline().strip()
+            else:
+                self.engine_info.app_name = "?"
+
+        return self.engine_info
+
+    def get_run_info(self) -> RunInfo:
+        if self.run_manager:
+            return self.run_manager.get_run_info()
+        else:
+            return None
+
+    def set_run_number(self, num):
+        # status = self.server.status
+        status = self.engine_info.status
+        if status == MachineStatus.STARTING or status == MachineStatus.STARTED:
+            return "run_number can not be changed during the FL app running."
+
+        run_folder = os.path.join(self.args.workspace, "run_" + str(num))
+        if os.path.exists(run_folder):
+            self.run_number = num
+            return "run number already exists. Set the FL run number to {}.".format(num)
+        else:
+            self.run_number = num
+            os.makedirs(run_folder)
+            return "Created a new run folder: run_{}".format(num)
+
+    def delete_run_number(self, num):
+        run_number_folder = os.path.join(self.args.workspace, "run_" + str(num))
+        if os.path.exists(run_number_folder):
+            shutil.rmtree(run_number_folder)
+        return ""
+
+    def get_run_number(self):
+        return self.run_number
+
+    def get_clients(self) -> [Client]:
+        return list(self.client_manager.clients.values())
+
+    def validate_clients(self, client_names: List[str]) -> Tuple[List[Client], List[str]]:
+        return self._get_all_clients_from_inputs(client_names)
+
+    def start_app_on_server(self) -> str:
+        if self.run_number == -1:
+            return "Please set a run number."
+
+        status = self.engine_info.status
+        if status == MachineStatus.STARTING or status == MachineStatus.STARTED:
+            return "Server already starting or started."
+        else:
+            app_root = os.path.join(self._get_run_folder(), self._get_server_app_folder())
+            if not os.path.exists(app_root):
+                return "Server app does not exist. Please deploy the server app before starting."
+
+            self.engine_info.status = MachineStatus.STARTING
+            if self.server.enable_byoc:
+                app_custom_folder = os.path.join(app_root, "custom")
+                try:
+                    sys.path.index(app_custom_folder)
+                except ValueError:
+                    self.remove_custom_path()
+                    sys.path.append(app_custom_folder)
+
+            # future = self.executor.submit(start_server_training, (self.server))
+            future = self.executor.submit(
+                lambda p: start_server_training(*p), [self.server, self.args, app_root, self.run_number]
+            )
+
+            start = time.time()
+            # Wait for the server App to start properly
+            while self.engine_info.status == MachineStatus.STARTING:
+                time.sleep(0.3)
+                if time.time() - start > 300.0:
+                    return "Could not start the server app."
+
+            if self.engine_info.status != MachineStatus.STARTED:
+                return f"Failed to start server app: {self.engine_info.status}"
+
+            return ""
+
+    def remove_custom_path(self):
+        regex = re.compile(".*/run_.*/custom")
+        custom_paths = list(filter(regex.search, sys.path))
+        for path in custom_paths:
+            sys.path.remove(path)
+
+    def abort_app_on_clients(self, clients: List[str]) -> str:
+        status = self.engine_info.status
+        if status == MachineStatus.STOPPED:
+            return "Server app has not started."
+        if status == MachineStatus.STARTING:
+            return "Server app is starting, please wait for started before abort."
+        return ""
+
+    def abort_app_on_server(self) -> str:
+        status = self.engine_info.status
+        if status == MachineStatus.STOPPED:
+            return "Server app has not started."
+        if status == MachineStatus.STARTING:
+            return "Server app is starting, please wait for started before abort."
+
+        self.logger.info("Abort the server app run.")
+        # self.server.stop_training()
+        self.server.shutdown = True
+        self.server.abort_run()
+        return ""
+
+    def check_app_start_readiness(self) -> str:
+        status = self.engine_info.status
+        if status != MachineStatus.STARTING and status != MachineStatus.STARTED:
+            return "Server app has not started."
+        return ""
+
+    def shutdown_server(self) -> str:
+        status = self.server.status
+        if status == ServerStatus.STARTING:
+            return "Server app is starting, please wait for started before shutdown."
+
+        self.logger.info("FL server shutdown.")
+        # self.server.fl_shutdown()
+        # future = self.executor.submit(server_shutdown, (self.server))
+
+        touch_file = os.path.join(self.args.workspace, "shutdown.fl")
+        future = self.executor.submit(lambda p: server_shutdown(*p), [self.server, touch_file])
+        return ""
+
+    def restart_server(self) -> str:
+        status = self.server.status
+        if status == ServerStatus.STARTING:
+            return "Server is starting, please wait for started before restart."
+
+        self.logger.info("FL server restart.")
+
+        touch_file = os.path.join(self.args.workspace, "restart.fl")
+        future = self.executor.submit(lambda p: server_shutdown(*p), [self.server, touch_file])
+        return ""
+
+    def get_widget(self, widget_id: str) -> Widget:
+        return self.widgets.get(widget_id)
+
+    def get_client_name_from_token(self, token: str) -> str:
+        client = self.server.client_manager.clients.get(token)
+        if client:
+            return client.name
+        else:
+            return ""
+
+    def get_all_clients(self):
+        return list(self.server.client_manager.clients.keys())
+
+    def _get_client_from_name(self, client_name):
+        for c in self.get_clients():
+            if client_name == c.name:
+                return c
+        return None
+
+    def _get_all_clients_from_inputs(self, inputs):
+        clients = []
+        invalid_inputs = []
+        all_clients = self.get_clients()
+        for item in inputs:
+            client = self.client_manager.clients.get(item)
+            # if item in self.get_all_clients():
+            if client:
+                clients.append(client)
+            else:
+                client = self._get_client_from_name(item)
+                if client:
+                    clients.append(client)
+                else:
+                    invalid_inputs.append(item)
+        return clients, invalid_inputs
+
+    def get_all_taskname(self) -> str:
+        return self.server.server_name
+
+    def get_app_data(self, app_name: str) -> Tuple[str, object]:
+        if self.run_number == -1:
+            return "Please set a FL run number.", None
+
+        # folder = self._get_run_folder()
+        # data = zip_directory_to_bytes(folder, self._get_client_app_folder(client_name))
+        fullpath_src = os.path.join(self.server.admin_server.file_upload_dir, app_name)
+        if not os.path.exists(fullpath_src):
+            return f"App folder '{app_name}' does not exist in staging area.", None
+
+        # folder = self.args.workspace
+        # data = zip_directory_to_bytes(
+        #     folder, os.path.join("run_" + str(self.run_number), self._get_client_app_folder(client_name))
+        # )
+        data = zip_directory_to_bytes(fullpath_src, "")
+        return "", data
+
+    def set_run_manager(self, run_manager: RunManager):
+        self.run_manager = run_manager
+        for _, widget in self.widgets.items():
+            self.run_manager.add_handler(widget)
+
+    def set_configurator(self, conf: ServerJsonConfigurator):
+        if not isinstance(conf, ServerJsonConfigurator):
+            raise TypeError("Must be a type of ServerJsonConfigurator.")
+        self.conf = conf
+
+    def build_component(self, config_dict):
+        return self.conf.build_component(config_dict)
+
+    def new_context(self) -> FLContext:
+        if self.run_manager:
+            return self.run_manager.new_context()
+        else:
+            return FLContext()
+
+    def get_component(self, component_id: str) -> object:
+        return self.run_manager.get_component(component_id)
+
+    def fire_event(self, event_type: str, fl_ctx: FLContext):
+        self.run_manager.fire_event(event_type, fl_ctx)
+
+    def get_staging_path_of_app(self, app_name: str) -> str:
+        return os.path.join(self.server.admin_server.file_upload_dir, app_name)
+
+    def deploy_app_to_server(self, app_name: str, app_staging_path: str) -> str:
+        return self.deploy_app(app_name, "app_server")
+
+    def prepare_deploy_app_to_client(self, app_name: str, app_staging_path: str, client_name: str) -> str:
+        return self.deploy_app(app_name, "app_" + client_name)
+
+    def get_workspace(self) -> Workspace:
+        return self.run_manager.get_workspace()
+
+    def ask_to_stop(self):
+        self.asked_to_stop = True
+
+    def deploy_app(self, src, dest):
+        if self.run_number == -1:
+            return "Please set a run number."
+
+        fullpath_src = os.path.join(self.server.admin_server.file_upload_dir, src)
+        fullpath_dest = os.path.join(self._get_run_folder(), dest)
+        if not os.path.exists(fullpath_src):
+            return f"App folder '{src}' does not exist in staging area."
+        if os.path.exists(fullpath_dest):
+            shutil.rmtree(fullpath_dest)
+        shutil.copytree(fullpath_src, fullpath_dest)
+
+        app_file = os.path.join(self._get_run_folder(), "fl_app.txt")
+        if os.path.exists(app_file):
+            os.remove(app_file)
+        with open(app_file, "wt") as f:
+            f.write(f"{src}")
+
+        return ""
+
+    def remove_clients(self, clients: List[str]) -> str:
+        for client in clients:
+            self._remove_dead_client(client)
+        return ""
+
+    def _remove_dead_client(self, token):
+        client = self.server.client_manager.remove_client(token)
+        # self.tokens.pop(token, None)
+        self.server.remove_client_data(token)
+        if self.server.admin_server:
+            self.server.admin_server.client_dead(client.name)
+
+    def register_aux_message_handler(self, topic: str, message_handle_func):
+        self.run_manager.aux_runner.register_aux_message_handler(topic, message_handle_func)
+
+    def send_aux_request(self, targets: [], topic: str, request: Shareable, timeout: float, fl_ctx: FLContext) -> dict:
+        if not targets:
+            targets = []
+            for t in self.get_clients():
+                targets.append(t.name)
+        if targets:
+            return self.run_manager.aux_runner.send_aux_request(
+                targets=targets, topic=topic, request=request, timeout=timeout, fl_ctx=fl_ctx
+            )
+        else:
+            return {}
+
+    def aux_send(self, targets: [], topic: str, request: Shareable, timeout: float, fl_ctx: FLContext) -> dict:
+        # Send the aux messages through admin_server
+        request.set_peer_props(fl_ctx.get_all_public_props())
+
+        message = Message(topic=ReservedTopic.AUX_COMMAND, body=pickle.dumps(request))
+        requests = {}
+        for n in targets:
+            requests.update({n: message})
+
+        replies = self.server.admin_server.send_requests(requests, timeout_secs=timeout)
+        results = {}
+        for r in replies:
+            client_name = self.get_client_name_from_token(r.client_token)
+            if r.reply:
+                try:
+                    results[client_name] = pickle.loads(r.reply.body)
+                except BaseException:
+                    results[client_name] = make_reply(ReturnCode.COMMUNICATION_ERROR)
+                    self.logger.error(
+                        f"Received unexpected reply from client: {client_name}, "
+                        f"message body:{r.reply.body} processing topic:{topic}"
+                    )
+            else:
+                results[client_name] = None
+
+        return results
+
+    def dispatch(self, topic: str, request: Shareable, fl_ctx: FLContext) -> Shareable:
+        return self.run_manager.aux_runner.dispatch(topic=topic, request=request, fl_ctx=fl_ctx)
+
+    def close(self):
+        self.executor.shutdown()
+
+
+def start_server_training(server, args, app_root, run_number):
+
+    restart_file = os.path.join(args.workspace, "restart.fl")
+    if os.path.exists(restart_file):
+        os.remove(restart_file)
+
+    if os.path.exists(os.path.join(app_root, args.env)):
+        env_config = args.env
+    else:
+        env_config = "/tmp/fl_server/environment.json"
+    try:
+        server_config_file_name = os.path.join(app_root, args.server_config)
+
+        conf = ServerJsonConfigurator(
+            config_file_name=server_config_file_name,
+        )
+        conf.configure()
+
+        set_up_run_config(server, conf)
+
+        server.start_run(run_number, app_root, conf, args)
+    except BaseException as e:
+        traceback.print_exc()
+        logging.getLogger().warning("FL server execution exception: " + str(e))
+    finally:
+        server.status = ServerStatus.STOPPED
+        server.engine.engine_info.status = MachineStatus.STOPPED
+        server.stop_training()
+        # if trainer:
+        #     trainer.close()
+
+        # Force garbage collection
+        gc.collect()
+
+    # return server.start()
+
+
+def server_shutdown(server, touch_file):
+    with open(touch_file, "a"):
+        os.utime(touch_file, None)
+
+    try:
+        server.fl_shutdown()
+        server.admin_server.stop()
+    finally:
+        sys.exit(2)
+
+
+def copy_new_server_properties(server, new_server):
+    # server.model_manager = new_server.model_manager
+    # server.model_saver = new_server.model_saver
+    server.builder = new_server.builder
+
+    server.wait_after_min_clients = new_server.wait_after_min_clients
+
+    server.outbound_filters = new_server.outbound_filters
+    server.inbound_filters = new_server.inbound_filters
+    server.cmd_modules = new_server.cmd_modules
+    server.processors = new_server.processors
+
+    # server.task_name = new_server.task_name
+    server.min_num_clients = new_server.min_num_clients
+    server.max_num_clients = new_server.max_num_clients
+    server.current_round = new_server.current_round
+    server.num_rounds = new_server.num_rounds
+    server.start_round = new_server.start_round
+
+    # server.heart_beat_timeout = new_server.heart_beat_timeout
+    # server.handlers = new_server.handlers
+
+    # clients = server.client_manager.clients
+    # server.client_manager = new_server.client_manager
+    # server.client_manager.clients = clients
+    server.client_manager.min_num_clients = new_server.client_manager.min_num_clients
+    server.client_manager.max_num_clients = new_server.client_manager.max_num_clients
+    server.client_manager.logger = new_server.client_manager.logger
+    server.client_manager.logger.disabled = False
+
+    server.reset_tokens()
+    server.contributed_clients.clear()
+    # server.accumulator.clear()
+
+    server.fl_ctx = new_server.fl_ctx
+
+    server.controller = new_server.controller
+    # server.model_aggregator = new_server.model_aggregator
+    # server.model_saver = new_server.model_saver
+    # server.shareable_generator = new_server.shareable_generator
+
+
+def set_up_run_config(server, conf):
+    server.heart_beat_timeout = conf.heartbeat_timeout
+    server.runner_config = conf.runner_config
+    server.handlers = conf.handlers
diff --git a/nvflare/private/fed/server/server_engine_internal_spec.py b/nvflare/private/fed/server/server_engine_internal_spec.py
new file mode 100644
index 0000000000..f47ccd09ea
--- /dev/null
+++ b/nvflare/private/fed/server/server_engine_internal_spec.py
@@ -0,0 +1,287 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from abc import ABC
+
+from nvflare.apis.fl_constant import MachineStatus
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.server_engine_spec import ServerEngineSpec
+from nvflare.apis.shareable import Shareable
+from .run_manager import RunManager, RunInfo
+from .server_json_config import ServerJsonConfigurator
+
+
+class EngineInfo(object):
+    def __init__(self):
+        self.start_time = time.time()
+        self.status = MachineStatus.STOPPED
+
+        self.app_name = "?"
+
+
+class ServerEngineInternalSpec(ServerEngineSpec, ABC):
+
+    # methods for internal use
+    def get_engine_info(self) -> EngineInfo:
+        """
+        Get general info of the Engine
+
+        Returns:
+
+        """
+        pass
+
+    def get_run_info(self) -> RunInfo:
+        pass
+
+    def get_staging_path_of_app(self, app_name: str) -> str:
+        """
+        Get the staging path of the app waiting to be deployed
+
+        Args:
+            app_name:
+
+        Returns: the app's folder path or empty string if the app doesn't exist
+
+        """
+        pass
+
+    def deploy_app_to_server(self, app_name: str, app_staging_path: str) -> str:
+        """
+        Deploy the specified app to the server.
+        Copy the app folder tree from staging area to the server's RUN area
+
+        Args:
+            app_name: name of the app to be deployed
+            app_staging_path: the full path to the app folder in staging area
+
+        Returns: error message if couldn't deploy, or empty string if successful
+
+        """
+        pass
+
+    def prepare_deploy_app_to_client(self, app_name: str, app_staging_path: str, client_name: str) -> str:
+        """
+        Prepare to deploy the specified app to the specified client name.
+        Copy the app folder tree from staging area to the client's RUN area on Server
+        Args:
+            app_name: name of the app
+            app_staging_path: the full path to the app folder in staging area
+            client_name: name of the client
+
+        Returns: error message if couldn't deploy, or empty string if successful
+
+        """
+        pass
+
+    def get_app_data(self, app_name: str) -> (str, object):
+        """
+        Get data for deploying the app
+
+        Args:
+            client_name:
+
+        Returns: error msg and data object.
+
+        """
+        pass
+
+    def get_run_number(self) -> str:
+        """
+        Get the current run_number of the engine
+        Returns:run_number
+
+        """
+        pass
+
+    def set_run_number(self, run_num: int) -> str:
+        """
+        Set the run number for the next RUN.
+
+        Args:
+            run_num:
+
+        Returns: error if any
+
+        """
+        pass
+
+    def delete_run_number(self, run_num: int) -> str:
+        """
+        Delete specified RUN.
+
+        The Engine must do status check before the run can be deleted.
+        Args:
+            run_num:
+
+        Returns: error if any
+
+        """
+        pass
+
+    def start_app_on_server(self) -> str:
+        """
+        Start the FL app on Server.
+
+        Returns: error if any
+
+        """
+        pass
+
+    def check_app_start_readiness(self) -> str:
+        """
+        Check whether the app is ready to start.
+
+        Returns: error if any
+
+        """
+        pass
+
+    def abort_app_on_clients(self, clients: [str]):
+        """
+        Clean up the specified client for aborting
+
+        Returns:
+
+        """
+        pass
+
+    def abort_app_on_server(self):
+        """
+        Clean up the app on Server
+
+        Returns:
+
+        """
+        pass
+
+    def shutdown_server(self) -> str:
+        """
+        Shutdown the FL Server
+
+        Returns: error if any
+
+        The engine should not exit right away.
+        It should set its status to STOPPING, and set up a timer (in a different thread),
+        and return from this call right away (if other restart conditions are met).
+        When the timer fires, it exits.
+        This would give the caller to process the feedback or clean up (e.g. admin cmd response).
+
+        """
+        pass
+
+    def remove_clients(self, clients: [str]) -> str:
+        """
+        Remove specified clients
+
+        Args:
+            clients:
+
+        Returns: error if any
+
+        """
+        pass
+
+    def restart_server(self) -> str:
+        """
+        Restart the FL Server
+
+        Returns: error if any
+
+        The engine should not exit right away.
+        See shutdown_server.
+
+        """
+        pass
+
+    def set_run_manager(self, run_manager: RunManager):
+        """
+        set the RunManager for FL server
+        Args:
+            run_manager:
+
+        Returns:
+
+        """
+        pass
+
+    def set_configurator(self, conf: ServerJsonConfigurator):
+        """
+        set the ServerJsonConfigurator for FL server
+        Args:
+            conf:
+
+        Returns:
+
+        """
+        pass
+
+    def build_component(self, config_dict):
+        """
+        Build a component from the config_dict.
+        Args:
+            config_dict:
+
+        Returns:
+
+        """
+        pass
+
+    def get_client_name_from_token(self, token: str) -> str:
+        """
+        Get the registered client name from communication token.
+        Args:
+            token:
+
+        Returns: client name
+
+        """
+        pass
+
+    def ask_to_stop(self):
+        """
+        Ask the engine to stop the current run.
+        Returns:
+
+        """
+        pass
+
+    def aux_send(
+            self,
+            targets: [],
+            topic: str,
+            request: Shareable,
+            timeout: float,
+            fl_ctx: FLContext
+    ) -> dict:
+        """
+        Send a request to client(s) via the aux channel
+
+        Args:
+            targets: list of Client or client names
+            topic: topic of the request
+            request: request to be sent
+            timeout: number of secs to wait for replies
+            fl_ctx: FL context
+
+        Returns: a MessageSendStatus and a dict of replies: client_name => Shareable
+
+        NOTE: when a reply is received, the peer_ctx props must be set into the PEER_PROPS header
+        of the reply Shareable.
+
+        If a reply is not received from a client, do not put it into the reply dict.
+
+        """
+        pass
diff --git a/nvflare/private/fed/server/server_json_config.py b/nvflare/private/fed/server/server_json_config.py
new file mode 100644
index 0000000000..1e0dfc6ff3
--- /dev/null
+++ b/nvflare/private/fed/server/server_json_config.py
@@ -0,0 +1,135 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.responder import Responder
+from nvflare.fuel.utils.json_scanner import Node
+from nvflare.private.fed_json_config import FedJsonConfigurator
+from nvflare.private.json_configer import ConfigContext, ConfigError
+from .server_runner import ServerRunnerConfig
+
+FL_PACKAGES = ["nvflare"]
+FL_MODULES = ["server", "client", "aggregators", "handlers", "pt", "app", "app_common", "workflows"]
+
+
+class WorkFlow:
+
+    def __init__(self, id, responder: Responder):
+        super().__init__()
+        self.id = id
+        self.responder = responder
+
+
+class ServerJsonConfigurator(FedJsonConfigurator):
+    def __init__(
+        self,
+        config_file_name: str,
+        exclude_libs=True
+    ):
+        base_pkgs = FL_PACKAGES
+        module_names = FL_MODULES
+
+        FedJsonConfigurator.__init__(
+            self,
+            config_file_name=config_file_name,
+            base_pkgs=base_pkgs,
+            module_names=module_names,
+            exclude_libs=exclude_libs)
+
+        self.runner_config = None
+
+        # if server doesn't hear heartbeat from client for this long, we'll consider the client dead
+        self.heartbeat_timeout = 60      # default to 1 minute
+
+        # server will ask client to come back for next task after this many secs
+        self.task_request_interval = 2   # default to 2 secs
+
+        # workflows to be executed
+        self.workflows = []
+
+    def process_config_element(self, config_ctx: ConfigContext, node: Node):
+        FedJsonConfigurator.process_config_element(self, config_ctx, node)
+
+        element = node.element
+        path = node.path()
+
+        if path == "server.heart_beat_timeout":
+            self.heartbeat_timeout = element
+            if not isinstance(element, int) and not isinstance(element, float):
+                raise ConfigError('"heart_beat_timeout" must be a number, but got {}'.format(type(element)))
+
+            if element <= 0.0:
+                raise ConfigError('"heart_beat_timeout" must be positive number, but got {}'.format(element))
+
+            return
+
+        if path == "server.task_request_interval":
+            self.task_request_interval = element
+            if not isinstance(element, int) and not isinstance(element, float):
+                raise ConfigError('"task_request_interval" must be a number, but got {}'.format(type(element)))
+
+            if element < 1:
+                raise ConfigError('"task_request_interval" must >= 1, but got {}'.format(element))
+
+            return
+
+        if re.search(r"^workflows\.#[0-9]+$", path):
+            workflow = self.build_component(element)
+            if not isinstance(workflow, Responder):
+                raise ConfigError('"workflow" must be a Responder or Controller object, but got {}'.format(
+                    type(workflow)))
+
+            cid = element.get("id", None)
+            if not cid:
+                cid = type(workflow).__name__
+
+            if not isinstance(cid, str):
+                raise ConfigError('"id" must be str but got {}'.format(type(cid)))
+
+            if cid in self._get_all_workflows_ids():
+                raise ConfigError('duplicate workflow id "{}"'.format(cid))
+
+            self.workflows.append(WorkFlow(cid, workflow))
+            return
+
+    def _get_all_workflows_ids(self):
+        ids = []
+        for t in self.workflows:
+            ids.append(t.id)
+        return ids
+
+    def build_component(self, config_dict):
+        t = super().build_component(config_dict)
+        if isinstance(t, FLComponent):
+            if type(t).__name__ not in [type(h).__name__ for h in self.handlers]:
+                self.handlers.append(t)
+        return t
+
+    def finalize_config(self, config_ctx: ConfigContext):
+        FedJsonConfigurator.finalize_config(self, config_ctx)
+
+        if not self.workflows:
+            raise ConfigError('workflows not specified')
+
+        self.runner_config = ServerRunnerConfig(
+            heartbeat_timeout=self.heartbeat_timeout,
+            task_request_interval=self.task_request_interval,
+            workflows=self.workflows,
+            task_data_filters=self.data_filter_table,
+            task_result_filters=self.result_filter_table,
+            components=self.components,
+            handlers=self.handlers
+        )
diff --git a/nvflare/private/fed/server/server_runner.py b/nvflare/private/fed/server/server_runner.py
new file mode 100644
index 0000000000..17c2678775
--- /dev/null
+++ b/nvflare/private/fed/server/server_runner.py
@@ -0,0 +1,420 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+
+from nvflare.apis.client import Client
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey, ReservedKey, ReservedTopic
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.fl_exception import WorkflowError
+from nvflare.apis.server_engine_spec import ServerEngineSpec
+from nvflare.apis.signal import Signal
+from nvflare.apis.shareable import Shareable, ReservedHeaderKey
+from nvflare.private.defs import SpecialTaskName, TaskConstant
+from nvflare.widgets.info_collector import GroupInfoCollector, InfoCollector
+
+
+class ServerRunnerConfig(object):
+    def __init__(
+        self,
+        heartbeat_timeout: int,
+        task_request_interval: int,
+        workflows: [],
+        task_data_filters: dict,  # task_name => list of filters
+        task_result_filters: dict,  # task_name => list of filters
+        handlers=None,  # list of event handlers
+        components=None,  # dict of extra python objects: id => object
+    ):
+        self.heartbeat_timeout = heartbeat_timeout
+        self.task_request_interval = task_request_interval
+        self.workflows = workflows
+        self.task_data_filters = task_data_filters
+        self.task_result_filters = task_result_filters
+        self.handlers = handlers
+        self.components = components
+
+
+class ServerRunner(FLComponent):
+    def __init__(self, config: ServerRunnerConfig, run_num: int, engine: ServerEngineSpec):
+        FLComponent.__init__(self)
+        self.run_num = run_num
+        self.config = config
+        self.engine = engine
+        self.abort_signal = Signal()
+        self.wf_lock = threading.Lock()
+        self.current_wf = None
+        self.status = "init"
+
+    def _execute_run(self):
+        for wf in self.config.workflows:
+            try:
+                with self.engine.new_context() as fl_ctx:
+                    self.log_info(fl_ctx, "starting workflow {} ({}) ...".format(wf.id, type(wf.responder)))
+
+                    wf.responder.initialize_run(fl_ctx)
+
+                    self.log_info(fl_ctx, "Workflow {} ({}) started".format(wf.id, type(wf.responder)))
+                    fl_ctx.set_prop(FLContextKey.WORKFLOW, wf.id, sticky=True)
+                    self.log_debug(fl_ctx, "firing event EventType.START_WORKFLOW")
+                    self.fire_event(EventType.START_WORKFLOW, fl_ctx)
+
+                    # use the wf_lock to ensure state integrity between workflow change and message processing
+                    with self.wf_lock:
+                        # we only set self.current_wf to open for business after successful initialize_run!
+                        self.current_wf = wf
+
+                with self.engine.new_context() as fl_ctx:
+                    wf.responder.control_flow(self.abort_signal, fl_ctx)
+            except WorkflowError as e:
+                with self.engine.new_context() as fl_ctx:
+                    self.log_exception(fl_ctx, "Fatal error occurred in workflow {}. Aborting the RUN".format(wf.id))
+                self.abort_signal.trigger(True)
+            except BaseException as e:
+                with self.engine.new_context() as fl_ctx:
+                    self.log_exception(fl_ctx, "exception in workflow {}".format(wf.id))
+            finally:
+                with self.engine.new_context() as fl_ctx:
+                    # do not execute finalize_run() until the wf_lock is acquired
+                    with self.wf_lock:
+                        # unset current_wf to prevent message processing
+                        # then we can release the lock - no need to delay message processing
+                        # during finalization!
+                        # Note: WF finalization may take time since it needs to wait for
+                        # the job monitor to join.
+                        self.current_wf = None
+
+                    self.log_info(fl_ctx, f"Workflow: {wf.id} finalizing ...")
+                    try:
+                        wf.responder.finalize_run(fl_ctx)
+                    except:
+                        self.log_exception(
+                            fl_ctx,
+                            'error finalizing workflow {}'.format(wf.id))
+
+                    self.log_debug(fl_ctx, "firing event EventType.END_WORKFLOW")
+                    self.fire_event(EventType.END_WORKFLOW, fl_ctx)
+
+                # Stopped the server runner from the current responder, not continue the following responders.
+                if self.abort_signal.triggered:
+                    break
+
+    def run(self):
+        with self.engine.new_context() as fl_ctx:
+            self.log_info(fl_ctx, "Server runner starting ...")
+
+            self.log_debug(fl_ctx, "firing event EventType.START_RUN")
+            fl_ctx.set_prop(ReservedKey.RUN_ABORT_SIGNAL, self.abort_signal,
+                            private=True, sticky=True)
+            self.fire_event(EventType.START_RUN, fl_ctx)
+
+        self.status = "started"
+        try:
+            self._execute_run()
+        except:
+            with self.engine.new_context() as fl_ctx:
+                self.log_exception(fl_ctx, 'Error executing RUN')
+
+        # use wf_lock to ensure state of current_wf!
+        self.status = "done"
+        with self.wf_lock:
+            with self.engine.new_context() as fl_ctx:
+                self.log_debug(fl_ctx, "firing event EventType.END_RUN")
+                self.fire_event(EventType.END_RUN, fl_ctx)
+
+        # ask all clients to end run!
+        self.engine.send_aux_request(
+            targets=None,
+            topic=ReservedTopic.END_RUN,
+            request=Shareable(),
+            timeout=0.,
+            fl_ctx=fl_ctx
+        )
+
+        self.log_info(fl_ctx, "Server runner finished.")
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == InfoCollector.EVENT_TYPE_GET_STATS:
+            collector = fl_ctx.get_prop(InfoCollector.CTX_KEY_STATS_COLLECTOR)
+            if collector:
+                assert isinstance(collector, GroupInfoCollector)
+
+                with self.wf_lock:
+                    if self.current_wf:
+                        collector.set_info(
+                            group_name="ServerRunner",
+                            info={"run_number": self.run_num,
+                                  "status": self.status,
+                                  "workflow": self.current_wf.id})
+        elif event_type == EventType.FATAL_SYSTEM_ERROR:
+            reason = fl_ctx.get_prop(key=FLContextKey.EVENT_DATA, default='')
+            self.log_error(fl_ctx,
+                           "Aborting current RUN due to FATAL_SYSTEM_ERROR received: {}".format(reason))
+            self.abort(fl_ctx)
+
+    def _task_try_again(self) -> (str, str, Shareable):
+        task = Shareable()
+        task[TaskConstant.WAIT_TIME] = self.config.task_request_interval
+        return SpecialTaskName.TRY_AGAIN, "", task
+
+    def process_task_request(self, client: Client, fl_ctx: FLContext) -> (str, str, Shareable):
+        """
+        Called by the Engine when a task request is received from a client.
+
+        NOTE: the Engine must create a new fl_ctx and:
+
+            with engine.new_context() as fl_ctx:
+                name, id, data = runner.process_task_request(client, fl_ctx)
+                ...
+
+        Args:
+            client:
+            fl_ctx:
+
+        Returns: task name, task id, and task data
+
+        """
+        engine = fl_ctx.get_engine()
+        assert isinstance(engine, ServerEngineSpec)
+
+        self.log_info(fl_ctx, "got task request from client")
+
+        if self.status == "init":
+            self.log_info(fl_ctx, "server runner still initializing - asked client to try again later")
+            return self._task_try_again()
+
+        if self.status == "done":
+            self.log_info(fl_ctx, "server runner is finalizing - asked client to end the run")
+            return SpecialTaskName.END_RUN, "", None
+
+        peer_ctx = fl_ctx.get_peer_context()
+        if not isinstance(peer_ctx, FLContext):
+            self.log_error(fl_ctx, "invalid task request: no peer context, asked client to try again.")
+            return self._task_try_again()
+
+        peer_run_num = peer_ctx.get_run_number()
+        if not peer_run_num or peer_run_num != self.run_num:
+            # the client is on a different RUN
+            self.log_info(fl_ctx, "invalid task request: not the same run_number, asked client to end run")
+            return SpecialTaskName.END_RUN, "", None
+
+        try:
+            with self.wf_lock:
+                if self.current_wf is None:
+                    self.log_info(fl_ctx, "There's no current workflow - asked client to try again later")
+                    return self._task_try_again()
+
+                task_name, task_id, task_data = self.current_wf.responder.process_task_request(client, fl_ctx)
+
+                if not task_name or task_name == SpecialTaskName.TRY_AGAIN:
+                    self.log_info(fl_ctx, "no task currently for client - asked client to try again later")
+                    return self._task_try_again()
+
+                if task_data:
+                    if not isinstance(task_data, Shareable):
+                        self.log_error(
+                            fl_ctx,
+                            "bad task data generated by workflow {}: must be Shareable but got {}".format(
+                                type(self.current_wf.id), type(task_data)
+                            ))
+                        return self._task_try_again()
+                else:
+                    task_data = Shareable()
+
+                task_data.set_header(ReservedHeaderKey.TASK_ID, task_id)
+                task_data.set_header(ReservedHeaderKey.TASK_NAME, task_name)
+                task_data.add_cookie(ReservedHeaderKey.WORKFLOW, self.current_wf.id)
+
+                if task_data:
+                    if not isinstance(task_data, Shareable):
+                        self.log_error(
+                            fl_ctx,
+                            "bad task data generated by workflow {}: must be Shareable but got {}".format(
+                                type(self.current_wf.id), type(task_data)
+                            ))
+                        return self._task_try_again()
+                else:
+                    task_data = Shareable()
+
+                task_data.set_header(ReservedHeaderKey.TASK_ID, task_id)
+                task_data.set_header(ReservedHeaderKey.TASK_NAME, task_name)
+                task_data.add_cookie(ReservedHeaderKey.WORKFLOW, self.current_wf.id)
+
+                if task_data:
+                    if not isinstance(task_data, Shareable):
+                        self.log_error(
+                            fl_ctx,
+                            "bad task data generated by workflow {}: must be Shareable but got {}".format(
+                                type(self.current_wf.id), type(task_data)
+                            ))
+                        return self._task_try_again()
+                else:
+                    task_data = Shareable()
+
+                task_data.set_header(ReservedHeaderKey.TASK_ID, task_id)
+                task_data.set_header(ReservedHeaderKey.TASK_NAME, task_name)
+                task_data.add_cookie(ReservedHeaderKey.WORKFLOW, self.current_wf.id)
+
+            self.log_info(fl_ctx, "assigned task to client: name={}, id={}".format(task_name, task_id))
+
+            # filter task data
+            fl_ctx.set_prop(FLContextKey.TASK_NAME, value=task_name, private=True, sticky=False)
+            fl_ctx.set_prop(FLContextKey.TASK_DATA, value=task_data, private=True, sticky=False)
+            fl_ctx.set_prop(FLContextKey.TASK_ID, value=task_id, private=True, sticky=False)
+
+            self.log_debug(fl_ctx, "firing event EventType.BEFORE_TASK_DATA_FILTER")
+            self.fire_event(EventType.BEFORE_TASK_DATA_FILTER, fl_ctx)
+            filter_list = self.config.task_data_filters.get(task_name)
+            if filter_list:
+                for f in filter_list:
+                    try:
+                        task_data = f.process(task_data, fl_ctx)
+                    except BaseException as ex:
+                        self.log_exception(
+                            fl_ctx,
+                            "processing error in task data filter {}; asked client to try again later".format(type(f)),
+                        )
+
+                        with self.wf_lock:
+                            if self.current_wf:
+                                self.current_wf.responder.handle_exception(task_id, fl_ctx)
+                        return self._task_try_again()
+
+            self.log_debug(fl_ctx, "firing event EventType.AFTER_TASK_DATA_FILTER")
+            self.fire_event(EventType.AFTER_TASK_DATA_FILTER, fl_ctx)
+            self.log_info(fl_ctx, "sent task assignment to client")
+            return task_name, task_id, task_data
+        except BaseException:
+            self.log_exception(fl_ctx, "error processing client task request; asked client to try again later")
+            return self._task_try_again()
+
+    def process_submission(self, client: Client, task_name: str, task_id: str, result: Shareable, fl_ctx: FLContext):
+        """
+        Called by the Engine when a task result is received from a client.
+
+        NOTE: the Engine must create a new fl_ctx and:
+
+            with engine.new_context() as fl_ctx:
+                name, id, data = runner.process_submission(client, fl_ctx)
+
+        Args:
+            client:
+            task_name:
+            task_id:
+            result:
+            fl_ctx:
+
+        Returns:
+
+        """
+        self.log_info(fl_ctx, "got result from client for task: name={}, id={}".format(task_name, task_id))
+
+        if not isinstance(result, Shareable):
+            self.log_error(
+                fl_ctx,
+                "invalid result submission: must be Shareable but got {}".format(type(result)))
+            return
+
+        # set the reply prop so log msg context could include RC from it
+        fl_ctx.set_prop(FLContextKey.REPLY, result, private=True, sticky=False)
+
+        if not isinstance(result, Shareable):
+            self.log_error(
+                fl_ctx,
+                "invalid result submission: must be Shareable but got {}".format(type(result)))
+            return
+
+        # set the reply prop so log msg context could include RC from it
+        fl_ctx.set_prop(FLContextKey.REPLY, result, private=True, sticky=False)
+
+        if not isinstance(result, Shareable):
+            self.log_error(
+                fl_ctx,
+                "invalid result submission: must be Shareable but got {}".format(type(result)))
+            return
+
+        # set the reply prop so log msg context could include RC from it
+        fl_ctx.set_prop(FLContextKey.REPLY, result, private=True, sticky=False)
+
+        fl_ctx.set_prop(FLContextKey.TASK_NAME, value=task_name, private=True, sticky=False)
+        fl_ctx.set_prop(FLContextKey.TASK_RESULT, value=result, private=True, sticky=False)
+        fl_ctx.set_prop(FLContextKey.TASK_ID, value=task_id, private=True, sticky=False)
+
+        if self.status != "started":
+            self.log_info(fl_ctx, "ignored result submission since server runner is {}".format(self.status))
+            return
+
+        peer_ctx = fl_ctx.get_peer_context()
+        if not isinstance(peer_ctx, FLContext):
+            self.log_error(fl_ctx, "invalid result submission: no peer context; dropped.")
+            return
+
+        peer_run_num = peer_ctx.get_run_number()
+        if not peer_run_num or peer_run_num != self.run_num:
+            # the client is on a different RUN
+            self.log_error(fl_ctx, "invalid result submission: not the same run number; dropped")
+            return
+
+        result.set_header(ReservedHeaderKey.TASK_NAME, task_name)
+        result.set_header(ReservedHeaderKey.TASK_ID, task_id)
+        result.set_peer_props(peer_ctx.get_all_public_props())
+
+        # filter task result
+        self.log_debug(fl_ctx, "firing event EventType.BEFORE_TASK_RESULT_FILTER")
+        self.fire_event(EventType.BEFORE_TASK_RESULT_FILTER, fl_ctx)
+        filter_list = self.config.task_result_filters.get(task_name)
+        if filter_list:
+            for f in filter_list:
+                try:
+                    result = f.process(result, fl_ctx)
+                except BaseException:
+                    self.log_exception(fl_ctx, "processing error in task result filter {}".format(type(f)))
+                    return
+
+        self.log_debug(fl_ctx, "firing event EventType.AFTER_TASK_RESULT_FILTER")
+        self.fire_event(EventType.AFTER_TASK_RESULT_FILTER, fl_ctx)
+
+        with self.wf_lock:
+            try:
+                if self.current_wf is None:
+                    self.log_info(fl_ctx, "There's no current workflow - dropped submission.")
+                    return
+
+                wf_id = result.get_cookie(ReservedHeaderKey.WORKFLOW, None)
+                if wf_id is not None and wf_id != self.current_wf.id:
+                    self.log_info(
+                        fl_ctx,
+                        "Got result for workflow {}, but we are running {} - dropped submission.".format(
+                            wf_id, self.current_wf.id
+                        ))
+                    return
+
+                self.log_debug(fl_ctx, "firing event EventType.BEFORE_PROCESS_SUBMISSION")
+                self.fire_event(EventType.BEFORE_PROCESS_SUBMISSION, fl_ctx)
+
+                self.current_wf.responder.process_submission(
+                    client=client, task_name=task_name, task_id=task_id, result=result, fl_ctx=fl_ctx
+                )
+                self.log_info(fl_ctx, "finished processing client result by {}".format(self.current_wf.id))
+
+                self.log_debug(fl_ctx, "firing event EventType.AFTER_PROCESS_SUBMISSION")
+                self.fire_event(EventType.AFTER_PROCESS_SUBMISSION, fl_ctx)
+            except BaseException:
+                self.log_exception(fl_ctx, "error processing client result by {}".format(self.current_wf.id))
+
+    def abort(self, fl_ctx: FLContext):
+        self.status = "done"
+        self.abort_signal.trigger(value=True)
+        self.log_info(fl_ctx, "asked to abort - triggered abort_signal to stop the RUN")
diff --git a/nvflare/private/fed/server/server_status.py b/nvflare/private/fed/server/server_status.py
new file mode 100644
index 0000000000..b1d793dadb
--- /dev/null
+++ b/nvflare/private/fed/server/server_status.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class ServerStatus(object):
+    NOT_STARTED = 0
+    STARTING = 1
+    STARTED = 2
+    STOPPED = 3
+
+    status_messages = {
+        NOT_STARTED: "app server not started",
+        STARTING: "app server starting",
+        STARTED: "app server started",
+        STOPPED: "app server stopped",
+    }
+
+
+def get_status_message(status):
+    return ServerStatus.status_messages.get(status)
diff --git a/nvflare/private/fed/server/shell_cmd.py b/nvflare/private/fed/server/shell_cmd.py
new file mode 100644
index 0000000000..b6358a4db9
--- /dev/null
+++ b/nvflare/private/fed/server/shell_cmd.py
@@ -0,0 +1,264 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import subprocess
+from typing import List
+
+from nvflare.fuel.hci.cmd_arg_utils import join_args
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec
+from nvflare.fuel.hci.server.constants import ConnProps
+from nvflare.fuel.hci.shell_cmd_val import (
+    CatValidator,
+    GrepValidator,
+    HeadValidator,
+    LsValidator,
+    ShellCommandValidator,
+    TailValidator,
+)
+from nvflare.private.defs import SysCommandTopic
+from nvflare.private.fed.server.admin import ClientReply, new_message
+from nvflare.private.fed.server.server_engine_internal_spec import ServerEngineInternalSpec
+from nvflare.security.security import Action, FLAuthzContext
+
+
+class _CommandExecutor(object):
+    def __init__(self, cmd_name: str, validator: ShellCommandValidator):
+        self.cmd_name = cmd_name
+        self.validator = validator
+
+    def authorize_command(self, conn: Connection, args: List[str]):
+        if len(args) < 2:
+            conn.append_error("syntax error: missing target")
+            return False, None
+
+        shell_cmd_args = [self.cmd_name]
+        for a in args[2:]:
+            shell_cmd_args.append(a)
+
+        shell_cmd = join_args(shell_cmd_args)
+
+        result = None
+        if self.validator:
+            err, result = self.validator.validate(shell_cmd_args[1:])
+            if len(err) > 0:
+                conn.append_error(err)
+                return False, None
+
+        # validate the command and make sure file destinations are protected
+        err = self.validate_shell_command(shell_cmd_args, result)
+        if len(err) > 0:
+            conn.append_error(err)
+            return False, None
+
+        site_name = args[1]
+        authz_ctx = FLAuthzContext.new_authz_context(site_names=[site_name], actions=[Action.VIEW])
+        conn.set_prop("shell_cmd", shell_cmd)
+        return True, authz_ctx
+
+    def validate_shell_command(self, args: List[str], parse_result) -> str:
+        return ""
+
+    def execute_command(self, conn: Connection, args: List[str]):
+        authz_ctx = conn.get_prop(ConnProps.AUTHZ_CTX, None)
+        if not authz_ctx:
+            conn.append_error("program error: no authorization context")
+            return
+
+        assert isinstance(authz_ctx, FLAuthzContext)
+        target = authz_ctx.site_names[0]
+        shell_cmd = conn.get_prop("shell_cmd")
+        if target == "server":
+            # run the shell command on server
+            output = subprocess.getoutput(shell_cmd)
+            conn.append_string(output)
+            return
+
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+        clients, invalid_inputs = engine.validate_clients([target])
+        if len(invalid_inputs) > 0:
+            conn.append_error("invalid client: {}".format(target))
+            return
+
+        if len(clients) > 1:
+            conn.append_error("this command can only be applied to one client at a time")
+            return
+
+        valid_tokens = []
+        for c in clients:
+            valid_tokens.append(c.token)
+
+        req = new_message(conn=conn, topic=SysCommandTopic.SHELL, body=shell_cmd)
+        server = conn.server
+        reply = server.send_request_to_client(req, valid_tokens[0], timeout_secs=server.timeout)
+        if reply is None:
+            conn.append_error("no reply from client - timed out")
+            return
+
+        assert isinstance(reply, ClientReply)
+        conn.append_string(reply.reply.body)
+
+    def get_usage(self):
+        if self.validator:
+            return self.validator.get_usage()
+        else:
+            return ""
+
+
+class _NoArgCmdExecutor(_CommandExecutor):
+    def __init__(self, cmd_name: str):
+        _CommandExecutor.__init__(self, cmd_name, None)
+
+    def validate_shell_command(self, args: List[str], parse_result):
+        if len(args) != 1:
+            return "this command does not accept extra args"
+
+        return ""
+
+
+class _FileCmdExecutor(_CommandExecutor):
+    def __init__(
+        self,
+        cmd_name: str,
+        validator: ShellCommandValidator,
+        text_file_only: bool = True,
+        single_file_only: bool = True,
+        file_required: bool = True,
+    ):
+        _CommandExecutor.__init__(self, cmd_name, validator)
+        self.text_file_only = text_file_only
+        self.single_file_only = single_file_only
+        self.file_required = file_required
+
+    def validate_shell_command(self, args: List[str], parse_result):
+        if self.file_required:
+            if not hasattr(parse_result, "files"):
+                return "a file is required as an argument"
+            if self.single_file_only and len(parse_result.files) != 1:
+                return "only one file is allowed"
+
+            for f in parse_result.files:
+                assert isinstance(f, str)
+
+                if not re.match("^[A-Za-z0-9-._/]*$", f):
+                    return "unsupported file {}".format(f)
+
+                if f.startswith("/"):
+                    return "absolute path is not allowed"
+
+                paths = f.split("/")
+                for p in paths:
+                    if p == "..":
+                        return ".. in path name is not allowed"
+
+                if self.text_file_only:
+                    basename, file_extension = os.path.splitext(f)
+                    if file_extension not in [".txt", ".log", ".json", ".csv", ".sh", ".config", ".py"]:
+                        return (
+                            "this command cannot be applied to file {}. Only files with the following extensions "
+                            "are permitted: .txt, .log, .json, .csv, .sh, .config, .py".format(f)
+                        )
+
+        return ""
+
+
+class ShellCommandModule(CommandModule):
+    def __init__(self):
+        CommandModule.__init__(self)
+
+    def get_spec(self):
+        pwd_exe = _NoArgCmdExecutor("pwd")
+        ls_exe = _FileCmdExecutor(
+            "ls", LsValidator(), text_file_only=False, single_file_only=False, file_required=False
+        )
+        cat_exe = _FileCmdExecutor("cat", CatValidator())
+        head_exe = _FileCmdExecutor("head", HeadValidator())
+        tail_exe = _FileCmdExecutor("tail", TailValidator())
+        grep_exe = _FileCmdExecutor("grep", GrepValidator())
+        env_exe = _NoArgCmdExecutor("env")
+
+        return CommandModuleSpec(
+            name="sys",
+            cmd_specs=[
+                CommandSpec(
+                    name="pwd",
+                    description="print the name of work directory",
+                    usage="pwd target\n" + 'where target is "server" or client name\n' + pwd_exe.get_usage(),
+                    handler_func=pwd_exe.execute_command,
+                    authz_func=pwd_exe.authorize_command,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="ls",
+                    description="list files in work dir",
+                    usage="ls target [options] [files]\n "
+                    + 'where target is "server" or client name\n'
+                    + ls_exe.get_usage(),
+                    handler_func=ls_exe.execute_command,
+                    authz_func=ls_exe.authorize_command,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="cat",
+                    description="show content of a file",
+                    usage="cat target [options] fileName\n "
+                    + 'where target is "server" or client name\n'
+                    + cat_exe.get_usage(),
+                    handler_func=cat_exe.execute_command,
+                    authz_func=cat_exe.authorize_command,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="head",
+                    description="print the first 10 lines of a file",
+                    usage="head target [options] fileName\n "
+                    + 'where target is "server" or client name\n'
+                    + head_exe.get_usage(),
+                    handler_func=head_exe.execute_command,
+                    authz_func=head_exe.authorize_command,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="tail",
+                    description="print the last 10 lines of a file",
+                    usage="tail target [options] fileName\n "
+                    + 'where target is "server" or client name\n'
+                    + tail_exe.get_usage(),
+                    handler_func=tail_exe.execute_command,
+                    authz_func=tail_exe.authorize_command,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="grep",
+                    description="search for PATTERN in a file.",
+                    usage="grep target [options] PATTERN fileName\n "
+                    + 'where target is "server" or client name\n'
+                    + grep_exe.get_usage(),
+                    handler_func=grep_exe.execute_command,
+                    authz_func=grep_exe.authorize_command,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name="env",
+                    description="show system environment vars",
+                    usage="env target\n " + 'where target is "server" or client name\n' + env_exe.get_usage(),
+                    handler_func=env_exe.execute_command,
+                    authz_func=env_exe.authorize_command,
+                    visible=True,
+                ),
+            ],
+        )
diff --git a/nvflare/private/fed/server/sys_cmd.py b/nvflare/private/fed/server/sys_cmd.py
new file mode 100644
index 0000000000..b922a0732c
--- /dev/null
+++ b/nvflare/private/fed/server/sys_cmd.py
@@ -0,0 +1,95 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import psutil
+
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec
+from nvflare.private.defs import SysCommandTopic
+from nvflare.private.fed.server.admin import new_message
+from nvflare.private.fed.server.cmd_utils import CommandUtil
+
+
+class SystemCommandModule(CommandModule, CommandUtil):
+    def __init__(self, allowed_commands=None):
+        self.allowed_commands = allowed_commands
+
+    def get_spec(self):
+        return CommandModuleSpec(
+            name="sys",
+            cmd_specs=[
+                CommandSpec(
+                    name="sys_info",
+                    description="get the system info",
+                    usage="sys_info server|client <client-name> ...",
+                    handler_func=self.sys_info,
+                    authz_func=self.authorize_operate,
+                    visible=True,
+                ),
+            ],
+        )
+
+    def sys_info(self, conn: Connection, args: [str]):
+        if len(args) < 2:
+            conn.append_error("syntax error: missing site names")
+            return
+
+        target_type = args[1]
+        if target_type == self.TARGET_TYPE_SERVER:
+            infos = dict(psutil.virtual_memory()._asdict())
+
+            table = conn.append_table(["Metrics", "Value"])
+
+            for k, v in infos.items():
+                table.add_row([str(k), str(v)])
+            table.add_row(
+                ["available_percent", "%.1f" % (psutil.virtual_memory().available * 100 / psutil.virtual_memory().total)]
+            )
+            return
+
+        if target_type == self.TARGET_TYPE_CLIENT:
+            message = new_message(conn, topic=SysCommandTopic.SYS_INFO, body="")
+            replies = self.send_request_to_clients(conn, message)
+            self._process_replies(conn, replies)
+            return
+
+        conn.append_string("invalid target type {}. Usage: sys_info server|client <client-name>".format(target_type))
+
+    def _process_replies(self, conn, replies):
+        if not replies:
+            conn.append_error('no responses from clients')
+            return
+
+        for r in replies:
+            conn.append_string("Client " + r.client_name)
+
+            if r.reply:
+                try:
+                    infos = json.loads(r.reply.body)
+                    table = conn.append_table(["Metrics", "Value"])
+
+                    for k, v in infos.items():
+                        table.add_row([str(k), str(v)])
+                    table.add_row(
+                        [
+                            "available_percent",
+                            "%.1f" % (psutil.virtual_memory().available * 100 / psutil.virtual_memory().total),
+                        ]
+                    )
+                except BaseException:
+                    conn.append_string(": Bad replies")
+            else:
+                conn.append_string(": No replies")
diff --git a/nvflare/private/fed/server/training_cmds.py b/nvflare/private/fed/server/training_cmds.py
new file mode 100644
index 0000000000..6346029027
--- /dev/null
+++ b/nvflare/private/fed/server/training_cmds.py
@@ -0,0 +1,640 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import time
+from typing import List
+
+from nvflare.apis.client import Client
+from nvflare.apis.fl_constant import AdminCommandNames, MachineStatus
+from nvflare.fuel.hci.conn import Connection
+from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec
+from nvflare.private.defs import ClientStatusKey, RequestHeader, TrainingTopic
+from nvflare.private.fed.server.admin import new_message
+from nvflare.private.fed.server.server_engine_internal_spec import ServerEngineInternalSpec
+from nvflare.security.security import Action, FLAuthzContext
+
+from .app_authz import AppAuthzService
+from .cmd_utils import CommandUtil
+
+
+class TrainingCommandModule(CommandModule, CommandUtil):
+
+    APP_STAGING_PATH = "app_staging_path"
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    def get_spec(self):
+        return CommandModuleSpec(
+            name="training",
+            cmd_specs=[
+                CommandSpec(
+                    name=AdminCommandNames.SET_RUN_NUMBER,
+                    description="set the run number",
+                    usage="set_run_number number",
+                    handler_func=self.set_run_number,
+                    authz_func=self.authorize_set_run_number,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.DELETE_RUN_NUMBER,
+                    description="delete a run",
+                    usage="delete_run_number number",
+                    handler_func=self.delete_run_number,
+                    authz_func=self.authorize_set_run_number,
+                    visible=True,
+                    confirm="auth",
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.DEPLOY_APP,
+                    description="deploy FL app to client/server",
+                    usage="deploy_app server|client <client-name>|all",
+                    handler_func=self.deploy_app,
+                    authz_func=self.authorize_deploy_app,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.START_APP,
+                    description="start the FL app",
+                    usage="start_app server|client|all",
+                    handler_func=self.start_app,
+                    authz_func=self.authorize_train,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.CHECK_STATUS,
+                    description="check status of the FL server/client",
+                    usage="check_status server|client",
+                    handler_func=self.check_status,
+                    authz_func=self.authorize_view,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.ABORT,
+                    description="abort the FL app",
+                    usage="abort server|client|all",
+                    handler_func=self.abort_app,
+                    authz_func=self.authorize_train,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.ABORT_TASK,
+                    description="abort the client current task execution",
+                    usage="abort_task <client-name>",
+                    handler_func=self.abort_task,
+                    authz_func=self.authorize_abort_client,
+                    visible=True,
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.REMOVE_CLIENT,
+                    description="remove a FL client",
+                    usage="remove_client <client-name>",
+                    handler_func=self.remove_client,
+                    authz_func=self.authorize_remove_client,
+                    visible=True,
+                    confirm="auth",
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.SHUTDOWN,
+                    description="shutdown the FL server/client",
+                    usage="shutdown server|client|all",
+                    handler_func=self.shutdown,
+                    authz_func=self.authorize_operate,
+                    visible=True,
+                    confirm="auth",
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.RESTART,
+                    description="restart the FL server/client",
+                    usage="restart server|client|all",
+                    handler_func=self.restart,
+                    authz_func=self.authorize_operate,
+                    visible=True,
+                    confirm="auth",
+                ),
+                CommandSpec(
+                    name=AdminCommandNames.SET_TIMEOUT,
+                    description="set the admin commands timeout",
+                    usage="set_timeout seconds ",
+                    handler_func=self.set_timeout,
+                    authz_func=self.authorize_set_timeout,
+                    visible=True,
+                ),
+            ],
+        )
+
+    # Set Run Number
+    def authorize_set_run_number(self, conn: Connection, args: List[str]):
+        if len(args) < 2:
+            conn.append_error("syntax error: missing run number")
+            return False, None
+
+        try:
+            num = int(args[1])
+        except ValueError:
+            conn.append_error("run number must be an integer.")
+            return False, None
+
+        if num < 1:
+            conn.append_error("run number must be > 0.")
+            return False, None
+
+        return True, FLAuthzContext.new_authz_context(site_names=[self.SITE_SERVER], actions=[Action.TRAIN])
+
+    def set_run_number(self, conn: Connection, args: List[str]):
+        num = int(args[1])
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+        conn.append_string(engine.set_run_number(num))
+        self._set_run_number_clients(conn, num)
+
+    def _set_run_number_clients(self, conn: Connection, run_number) -> bool:
+        engine = conn.app_ctx
+        clients = engine.get_clients()
+        if clients:
+            valid_tokens = []
+            for c in clients:
+                valid_tokens.append(c.token)
+            conn.set_prop(self.TARGET_CLIENT_TOKENS, valid_tokens)
+
+            message = new_message(conn, topic=TrainingTopic.SET_RUN_NUMBER, body="")
+            message.set_header(RequestHeader.RUN_NUM, str(run_number))
+            replies = self.send_request_to_clients(conn, message)
+            self.process_replies_to_table(conn, replies)
+            return True
+
+    def delete_run_number(self, conn: Connection, args: List[str]):
+        num = int(args[1])
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        engine_info = engine.get_engine_info()
+
+        if engine_info.status == MachineStatus.STARTED or engine_info.status == MachineStatus.STARTING:
+            run_number = engine.get_run_number()
+            if run_number == num:
+                conn.append_error("Current running run_number can not be deleted.")
+                return
+
+        err = engine.delete_run_number(num)
+        if err:
+            conn.append_error(err)
+            return
+
+        # ask clients to delete this RUN
+        message = new_message(conn, topic=TrainingTopic.DELETE_RUN, body="")
+        message.set_header(RequestHeader.RUN_NUM, str(num))
+        clients = engine.get_clients()
+        if clients:
+            conn.set_prop(self.TARGET_CLIENT_TOKENS, [x.token for x in clients])
+            replies = self.send_request_to_clients(conn, message)
+            self.process_replies_to_table(conn, replies)
+
+        conn.append_success("")
+
+    # Deploy
+    def authorize_deploy_app(self, conn: Connection, args: List[str]):
+        if len(args) < 3:
+            conn.append_error("syntax error: missing target")
+            return False, None
+
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+        if engine.get_run_number() < 0:
+            conn.append_error("Please set a run number.")
+            return False, None
+
+        err = self.validate_command_targets(conn, args[2:])
+        if err:
+            conn.append_error(err)
+            return False, None
+
+        app_name = args[1]
+        app_staging_path = engine.get_staging_path_of_app(app_name)
+        if not app_staging_path:
+            conn.append_error("App {} does not exist. Please upload it first".format(app_name))
+            return False, None
+
+        conn.set_prop(self.APP_STAGING_PATH, app_staging_path)
+        target_type = args[2]
+
+        if target_type == self.TARGET_TYPE_SERVER:
+            sites = [self.SITE_SERVER]
+        else:
+            sites = []
+            client_names = conn.get_prop(self.TARGET_CLIENT_NAMES)
+            if client_names:
+                sites.extend(client_names)
+
+            if target_type == self.TARGET_TYPE_ALL:
+                sites.append(self.SITE_SERVER)
+
+        err, authz_ctx = AppAuthzService.authorize_deploy(app_staging_path, sites)
+        if err:
+            conn.append_error(err)
+            return False, None
+        else:
+
+            return True, authz_ctx
+
+    def _deploy_to_clients(self, conn: Connection, app_name, app_staging_path) -> bool:
+        # return True if successful
+        engine = conn.app_ctx
+        client_names = conn.get_prop(self.TARGET_CLIENT_NAMES)
+        # for client_name in client_names:
+        #     err = engine.prepare_deploy_app_to_client(app_name, app_staging_path, client_name)
+        #     if err:
+        #         conn.append_error(err)
+        #         return False
+
+        err, app_data = engine.get_app_data(app_name)
+        if err:
+            conn.append_error(err)
+            return False
+
+        message = new_message(conn, topic=TrainingTopic.DEPLOY, body=app_data)
+        message.set_header(RequestHeader.RUN_NUM, str(engine.get_run_number()))
+        message.set_header(RequestHeader.APP_NAME, app_name)
+        replies = self.send_request_to_clients(conn, message)
+        self.process_replies_to_table(conn, replies)
+        return True
+
+    def _deploy_to_server(self, conn, app_name, app_staging_path) -> bool:
+        # return True if successful
+        engine = conn.app_ctx
+        err = engine.deploy_app_to_server(app_name, app_staging_path)
+        if not err:
+            conn.append_string('deployed app "{}" to Server'.format(app_name))
+            return True
+        else:
+            conn.append_error(err)
+            return False
+
+    def deploy_app(self, conn: Connection, args: List[str]):
+        app_name = args[1]
+
+        target_type = conn.get_prop(self.TARGET_TYPE)
+        app_staging_path = conn.get_prop(self.APP_STAGING_PATH)
+        if target_type == self.TARGET_TYPE_SERVER:
+            if not self._deploy_to_server(conn, app_name, app_staging_path):
+                return
+        elif target_type == self.TARGET_TYPE_CLIENT:
+            if not self._deploy_to_clients(conn, app_name, app_staging_path):
+                return
+        else:
+            # all
+            success = self._deploy_to_server(conn, app_name, app_staging_path)
+            if success:
+                client_names = conn.get_prop(self.TARGET_CLIENT_NAMES, None)
+                if client_names:
+                    if not self._deploy_to_clients(conn, app_name, app_staging_path):
+                        return
+            else:
+                return
+        conn.append_success("")
+
+    # Start App
+    def _start_app_on_server(self, conn: Connection) -> bool:
+        engine = conn.app_ctx
+        err = engine.start_app_on_server()
+        if err:
+            conn.append_error(err)
+            return False
+        else:
+            conn.append_string("Server app is starting....")
+            return True
+
+    def _start_app_on_clients(self, conn: Connection) -> bool:
+        engine = conn.app_ctx
+        err = engine.check_app_start_readiness()
+        if err:
+            conn.append_error(err)
+            return False
+
+        run_info = engine.get_run_info()
+        message = new_message(conn, topic=TrainingTopic.START, body="")
+        message.set_header(RequestHeader.RUN_NUM, str(run_info.run_number))
+        replies = self.send_request_to_clients(conn, message)
+        self.process_replies_to_table(conn, replies)
+        return True
+
+    def start_app(self, conn: Connection, args: List[str]):
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        target_type = args[1]
+        if target_type == self.TARGET_TYPE_SERVER:
+            if not self._start_app_on_server(conn):
+                return
+        elif target_type == self.TARGET_TYPE_CLIENT:
+            if not self._start_app_on_clients(conn):
+                return
+        else:
+            # all
+            success = self._start_app_on_server(conn)
+
+            if success:
+                # engine_info = None
+                # start = time.time()
+                # # Wait for the server App to start properly
+                # while engine_info is None or engine_info.status != MachineStatus.STARTED:
+                #     time.sleep(0.3)
+                #     engine_info = engine.get_engine_info()
+                #     if time.time() - start > 60.0:
+                #         conn.append_error("Could not start the server app")
+                #         return
+                #
+                client_names = conn.get_prop(self.TARGET_CLIENT_NAMES, None)
+                if client_names:
+                    if not self._start_app_on_clients(conn):
+                        return
+        conn.append_success("")
+
+    # Abort App
+    def _abort_clients(self, conn, clients: List[str]) -> bool:
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        err = engine.abort_app_on_clients(clients)
+        if err:
+            conn.append_error(err)
+            return False
+
+        run_info = engine.get_run_info()
+        message = new_message(conn, topic=TrainingTopic.ABORT, body="")
+        if run_info:
+            message.set_header(RequestHeader.RUN_NUM, str(run_info.run_number))
+
+        # conn.set_prop(self.TARGET_CLIENT_NAMES, client_names)
+        replies = self.send_request_to_clients(conn, message)
+        self.process_replies_to_table(conn, replies)
+        return True
+
+    def abort_app(self, conn: Connection, args: List[str]):
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        target_type = args[1]
+        if target_type == self.TARGET_TYPE_SERVER or target_type == self.TARGET_TYPE_ALL:
+            conn.append_string("Trying to abort all clients before abort server ...")
+            clients = engine.get_clients()
+            if clients:
+                tokens = [c.token for c in clients]
+                conn.set_prop(
+                    self.TARGET_CLIENT_TOKENS, tokens
+                )  # need this because not set in validate_command_targets when target_type == self.TARGET_TYPE_SERVER
+                if not self._abort_clients(conn, clients=[c.token for c in clients]):
+                    return
+            err = engine.abort_app_on_server()
+            if err:
+                conn.append_error(err)
+                return
+            conn.append_string("Abort signal has been sent to the server app.")
+        elif target_type == self.TARGET_TYPE_CLIENT:
+            clients = conn.get_prop(self.TARGET_CLIENT_TOKENS)
+            if not clients:
+                conn.append_string("No clients to abort")
+                return
+            if not self._abort_clients(conn, clients):
+                return
+        conn.append_success("")
+
+    def abort_task(self, conn, clients: List[str]) -> str:
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        err = engine.abort_app_on_clients(clients)
+        if err:
+            conn.append_error(err)
+            return ""
+
+        run_info = engine.get_run_info()
+        message = new_message(conn, topic=TrainingTopic.ABORT_TASK, body="")
+        if run_info:
+            message.set_header(RequestHeader.RUN_NUM, str(run_info.run_number))
+
+        # conn.set_prop(self.TARGET_CLIENT_NAMES, client_names)
+        replies = self.send_request_to_clients(conn, message)
+        return self.process_replies_to_table(conn, replies)
+
+    # Shutdown
+    def _shutdown_app_on_server(self, conn: Connection) -> bool:
+        engine = conn.app_ctx
+        err = engine.shutdown_server()
+        if err:
+            conn.append_error(err)
+            return False
+        else:
+            conn.append_string("FL app has been shutdown.")
+            conn.append_shutdown("Bye bye")
+            return True
+
+    def _shutdown_app_on_clients(self, conn: Connection) -> bool:
+        engine = conn.app_ctx
+        message = new_message(conn, topic=TrainingTopic.SHUTDOWN, body="")
+        clients = conn.get_prop(self.TARGET_CLIENT_TOKENS, None)
+        if not clients:
+            conn.append_error("no clients to shutdown")
+            return False
+
+        err = engine.remove_clients(clients)
+        if err:
+            conn.append_error(err)
+            return False
+
+        replies = self.send_request_to_clients(conn, message)
+        self.process_replies_to_table(conn, replies)
+        return True
+
+    def shutdown(self, conn: Connection, args: List[str]):
+        target_type = args[1]
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        if target_type == self.TARGET_TYPE_SERVER:
+            if engine.get_clients():
+                conn.append_error("There are still active clients. Shutdown all clients first.")
+                return
+            if not self._shutdown_app_on_server(conn):
+                return
+        elif target_type == self.TARGET_TYPE_CLIENT:
+            if not self._shutdown_app_on_clients(conn):
+                return
+        else:
+            # all
+            if engine.get_clients():
+                conn.append_string("Trying to shutdown clients before server...")
+                success = self._shutdown_app_on_clients(conn)
+                if success:
+                    if not self._shutdown_app_on_server(conn):
+                        return
+            else:
+                if not self._shutdown_app_on_server(conn):
+                    return
+        conn.append_success("")
+
+    # Remove Clients
+    def authorize_remove_client(self, conn: Connection, args: List[str]):
+        if len(args) < 2:
+            conn.append_error("syntax error: missing site names")
+            return False, None
+
+        auth_args = [args[0], self.TARGET_TYPE_CLIENT]
+        auth_args.extend(args[1:])
+        return self.authorize_operate(conn, auth_args)
+
+    def authorize_abort_client(self, conn: Connection, args: List[str]):
+        auth_args = [args[0], self.TARGET_TYPE_CLIENT]
+        auth_args.extend(args[1:])
+        return self.authorize_operate(conn, auth_args)
+
+    def remove_client(self, conn: Connection, args: List[str]):
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+        clients = conn.get_prop(self.TARGET_CLIENT_TOKENS)
+        err = engine.remove_clients(clients)
+        if err:
+            conn.append_error(err)
+            return
+        conn.append_success("")
+
+    # Restart
+    def _restart_clients(self, conn, clients) -> str:
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+        engine.remove_clients(clients)
+        message = new_message(conn, topic=TrainingTopic.RESTART, body="")
+        replies = self.send_request_to_clients(conn, message)
+        return self.process_replies_to_string(conn, replies)
+
+    def restart(self, conn: Connection, args: List[str]):
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+
+        target_type = args[1]
+        if target_type == self.TARGET_TYPE_SERVER or target_type == self.TARGET_TYPE_ALL:
+
+            clients = engine.get_clients()
+            if clients:
+                conn.append_string("Trying to restart all clients before restarting server...")
+                tokens = [c.token for c in clients]
+                conn.set_prop(
+                    self.TARGET_CLIENT_TOKENS, tokens
+                )  # need this because not set in validate_command_targets when target_type == self.TARGET_TYPE_SERVER
+                response = self._restart_clients(conn, tokens)
+                conn.append_string(response)
+                # check with Isaac - no need to wait!
+                # time.sleep(5)
+
+            err = engine.restart_server()
+            if err:
+                conn.append_error(err)
+            else:
+                conn.append_string("Server scheduled for restart")
+        elif target_type == self.TARGET_TYPE_CLIENT:
+            clients = conn.get_prop(self.TARGET_CLIENT_TOKENS)
+            if not clients:
+                conn.append_error("no clients available")
+                return
+            else:
+                response = self._restart_clients(conn, clients)
+                conn.append_string(response)
+        conn.append_success("")
+
+    # Set Timeout
+    def authorize_set_timeout(self, conn: Connection, args: List[str]):
+        if len(args) != 2:
+            conn.append_error("syntax error: missing timeout")
+            return False, None
+
+        try:
+            num = float(args[1])
+        except ValueError:
+            conn.append_error("must provide the timeout value in seconds")
+            return False, None
+
+        if num <= 0:
+            conn.append_error("timeout must be > 0")
+            return False, None
+
+        return True, FLAuthzContext.new_authz_context(site_names=[self.SITE_SERVER], actions=[Action.TRAIN])
+
+    def set_timeout(self, conn: Connection, args: List[str]):
+        timeout = float(args[1])
+        server = conn.server
+        server.timeout = timeout
+        conn.append_string("admin command timeout has been set to: {}".format(timeout))
+        conn.append_success("")
+
+    # Check Status
+    # Need more discussion on what status to be shown
+    # Show EngineStatus
+    def check_status(self, conn: Connection, args: List[str]):
+        engine = conn.app_ctx
+        assert isinstance(engine, ServerEngineInternalSpec)
+        dest = args[1]
+        if dest == self.TARGET_TYPE_SERVER:
+            engine_info = engine.get_engine_info()
+            conn.append_string(f"FL_app name: {engine_info.app_name}")
+            conn.append_string(f"Engine status: {engine_info.status.value}")
+            run_info = engine.get_run_info()
+            if engine.get_run_number() < 0:
+                conn.append_string("Run number has not been set.")
+            else:
+                conn.append_string(f"Current run number: {engine.get_run_number()}")
+
+            clients = engine.get_clients()
+            conn.append_string("Registered clients: {} ".format(len(clients)))
+
+            if clients:
+                table = conn.append_table(["Client", "Token", "Last Connect Time"])
+                for c in clients:
+                    assert isinstance(c, Client)
+                    table.add_row([c.name, str(c.token), time.asctime(time.localtime(c.last_connect_time))])
+        elif dest == self.TARGET_TYPE_CLIENT:
+            message = new_message(conn, topic=TrainingTopic.CHECK_STATUS, body="")
+            replies = self.send_request_to_clients(conn, message)
+            self._process_status_replies(conn, replies)
+        else:
+            conn.append_error("invalid target type {}. Usage: check_status server|client ...".format(dest))
+
+    def _process_status_replies(self, conn, replies):
+        if not replies:
+            conn.append_error("no responses from clients")
+            return
+
+        engine = conn.app_ctx
+        table = conn.append_table(["client", "app_name", "run_number", "status"])
+        for r in replies:
+            run_num = "?"
+            status = "?"
+            app_name = "?"
+            client_name = engine.get_client_name_from_token(r.client_token)
+
+            if r.reply:
+                try:
+                    body = json.loads(r.reply.body)
+                    if r.reply and isinstance(body, dict):
+                        app_name = body.get(ClientStatusKey.APP_NAME, "?")
+                        run_num = body.get(ClientStatusKey.RUN_NUM, "?")
+                        status = body.get(ClientStatusKey.STATUS, "?")
+                except BaseException:
+                    self.logger.error("Bad reply from client")
+
+                table.add_row([client_name, app_name, run_num, status])
+            else:
+                table.add_row([client_name, app_name, run_num, "No Reply"])
diff --git a/nvflare/private/fed/utils/__init__.py b/nvflare/private/fed/utils/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/private/fed/utils/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/private/fed/utils/fed_utils.py b/nvflare/private/fed/utils/fed_utils.py
new file mode 100644
index 0000000000..a5ed9e4988
--- /dev/null
+++ b/nvflare/private/fed/utils/fed_utils.py
@@ -0,0 +1,42 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pickle
+
+from nvflare.apis.fl_context import FLContext
+from nvflare.private.fed.protos.federated_pb2 import ModelData
+from nvflare.private.fed.utils.numproto import bytes_to_proto
+
+
+def shareable_to_modeldata(shareable, fl_ctx):
+    # make_init_proto message
+    model_data = ModelData()  # create an empty message
+
+    model_data.params["data"].CopyFrom(make_shareeable_data(shareable))
+
+    context_data = make_context_data(fl_ctx)
+    model_data.params["fl_context"].CopyFrom(context_data)
+    return model_data
+
+
+def make_shareeable_data(shareable):
+    return bytes_to_proto(shareable.to_bytes())
+
+
+def make_context_data(fl_ctx):
+    shared_fl_ctx = FLContext()
+    shared_fl_ctx.set_public_props(fl_ctx.get_all_public_props())
+    props = pickle.dumps(shared_fl_ctx)
+    context_data = bytes_to_proto(props)
+    return context_data
diff --git a/nvflare/private/fed/utils/messageproto.py b/nvflare/private/fed/utils/messageproto.py
new file mode 100644
index 0000000000..5019397740
--- /dev/null
+++ b/nvflare/private/fed/utils/messageproto.py
@@ -0,0 +1,49 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.private.admin_defs import Message
+from nvflare.private.fed.protos.admin_pb2 import Message as Proto_Message
+
+
+def message_to_proto(message: Message) -> Proto_Message:
+    proto_message = Proto_Message()
+    proto_message.id = message.id
+    proto_message.topic = message.topic
+    if isinstance(message.body, str):
+        proto_message.body_type = "str"
+        proto_message.body = bytes(message.body, "utf-8")
+    elif isinstance(message.body, bytes):
+        proto_message.body_type = "bytes"
+        proto_message.body = message.body
+    else:
+        proto_message.body_type = "unknown"
+        proto_message.body = message.body
+
+    for k, v in message.headers.items():
+        proto_message.headers[k] = v
+    return proto_message
+
+
+def proto_to_message(proto: Proto_Message) -> Message:
+    if proto.body_type == "str":
+        message = Message(topic=proto.topic, body=proto.body.decode("utf-8"))
+    elif proto.body_type == "bytes":
+        message = Message(topic=proto.topic, body=proto.body)
+    else:
+        message = Message(topic=proto.topic, body=proto.body)
+
+    message.id = proto.id
+    for k, v in proto.headers.items():
+        message.headers[k] = v
+    return message
diff --git a/nvflare/private/fed/utils/numproto.py b/nvflare/private/fed/utils/numproto.py
new file mode 100644
index 0000000000..fcc2448ec3
--- /dev/null
+++ b/nvflare/private/fed/utils/numproto.py
@@ -0,0 +1,68 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""NumPy ndarray to protobuf serialization and deserialization"""
+from io import BytesIO
+
+import numpy as np
+
+from nvflare.private.fed.protos.federated_pb2 import NDArray
+
+
+def ndarray_to_proto(nda: np.ndarray) -> NDArray:
+    """Serializes a numpy array into an NDArray protobuf message.
+    Args:
+        nda (np.ndarray): numpy array to serialize.
+    Returns:
+        Returns an NDArray protobuf message.
+    """
+    nda_bytes = BytesIO()
+    np.save(nda_bytes, nda, allow_pickle=False)
+
+    return NDArray(ndarray=nda_bytes.getvalue())
+
+
+def proto_to_ndarray(nda_proto: NDArray) -> np.ndarray:
+    """Deserializes an NDArray protobuf message into a numpy array.
+    Args:
+        nda_proto (NDArray): NDArray protobuf message to deserialize.
+    Returns:
+        Returns a numpy.ndarray.
+    """
+    nda_bytes = BytesIO(nda_proto.ndarray)
+
+    return np.load(nda_bytes, allow_pickle=False)
+
+
+def bytes_to_proto(data: bytes) -> NDArray:
+    """Serializes a bytes into an NDArray protobuf message.
+    Args:
+        data : bytes data
+    Returns:
+        Returns an NDArray protobuf message.
+    """
+    assert isinstance(data, bytes)
+    return NDArray(ndarray=data)
+
+
+def proto_to_bytes(nda_proto: NDArray) -> bytes:
+    """Deserializes an NDArray protobuf message into bytes.
+    Args:
+        nda_proto (NDArray): bytes.
+    Returns:
+        Returns bytes.
+    """
+    nda_bytes = BytesIO(nda_proto.ndarray)
+
+    return nda_bytes.read()
diff --git a/nvflare/private/fed_json_config.py b/nvflare/private/fed_json_config.py
new file mode 100644
index 0000000000..446e676504
--- /dev/null
+++ b/nvflare/private/fed_json_config.py
@@ -0,0 +1,172 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from nvflare.apis.filter import Filter
+from nvflare.fuel.utils.json_scanner import Node
+from nvflare.private.json_configer import JsonConfigurator, ConfigContext, ConfigError
+
+
+class FilterChain(object):
+    def __init__(self):
+        self.tasks = []
+        self.filters = []
+
+
+class FedJsonConfigurator(JsonConfigurator):
+    def __init__(self, config_file_name: str, base_pkgs: [str], module_names: [str], exclude_libs=True):
+        JsonConfigurator.__init__(
+            self,
+            config_file_name=config_file_name,
+            base_pkgs=base_pkgs,
+            module_names=module_names,
+            exclude_libs=exclude_libs,
+        )
+
+        self.format_version = None
+        self.handlers = []
+        self.components = {}  # id => component
+        self.task_data_filter_chains = []
+        self.task_result_filter_chains = []
+        self.current_filter_chain = None
+        self.data_filter_table = None
+        self.result_filter_table = None
+
+    def process_config_element(self, config_ctx: ConfigContext, node: Node):
+        element = node.element
+        path = node.path()
+
+        if path == "format_version":
+            self.format_version = element
+            return
+
+        # if re.search(r"^handlers\.#[0-9]+$", path):
+        #     h = self.build_component(element)
+        #     if not isinstance(h, FLComponent):
+        #         raise ConfigError("handler must be a FLComponent object, but got {}".format(type(h)))
+        #     # Ensure only add one instance of the handlers for the same component
+        #     if type(h).__name__ not in [type(t).__name__ for t in self.handlers]:
+        #         self.handlers.append(h)
+        #     return
+
+        if re.search(r"^components\.#[0-9]+$", path):
+            c = self.build_component(element)
+            cid = element.get("id", None)
+            if not cid:
+                raise ConfigError("missing component id")
+
+            if not isinstance(cid, str):
+                raise ConfigError('"id" must be str but got {}'.format(type(cid)))
+
+            if cid in self.components:
+                raise ConfigError('duplicate component id "{}"'.format(cid))
+
+            self.components[cid] = c
+            return
+
+        # result filters
+        if re.search(r"^task_result_filters\.#[0-9]+$", path):
+            self.current_filter_chain = FilterChain()
+            node.props["data"] = self.current_filter_chain
+            node.exit_cb = self._process_result_filter_chain
+            return
+
+        if re.search(r"^task_result_filters\.#[0-9]+\.tasks$", path):
+            self.current_filter_chain.tasks = element
+            return
+
+        if re.search(r"^task_result_filters.#[0-9]+\.filters\.#[0-9]+$", path):
+            f = self.build_component(element)
+            self.current_filter_chain.filters.append(f)
+            return
+
+        # data filters
+        if re.search(r"^task_data_filters\.#[0-9]+$", path):
+            self.current_filter_chain = FilterChain()
+            node.props["data"] = self.current_filter_chain
+            node.exit_cb = self._process_data_filter_chain
+            return
+
+        if re.search(r"^task_data_filters\.#[0-9]+\.tasks$", path):
+            self.current_filter_chain.tasks = element
+            return
+
+        if re.search(r"^task_data_filters.#[0-9]+\.filters\.#[0-9]+$", path):
+            f = self.build_component(element)
+            self.current_filter_chain.filters.append(f)
+            return
+
+    def validate_tasks(self, tasks):
+        if not isinstance(tasks, list):
+            raise ConfigError('"tasks" must be specified as list of task names')
+
+        if len(tasks) <= 0:
+            raise ConfigError('"tasks" must not be empty')
+
+        for n in tasks:
+            if not isinstance(n, str):
+                raise ConfigError("task names must be string but got {}".format(type(n)))
+
+    def validate_filter_chain(self, chain: FilterChain):
+        self.validate_tasks(chain.tasks)
+
+        if not isinstance(chain.filters, list):
+            raise ConfigError('"filters" must be specified as list of filters')
+
+        if len(chain.filters) <= 0:
+            raise ConfigError('"filters" must not be empty')
+
+        for f in chain.filters:
+            if not isinstance(f, Filter):
+                raise ConfigError('"filters" must contain Filter object but got {}'.format(type(f)))
+
+    def _process_result_filter_chain(self, node: Node):
+        filter_chain = node.props["data"]
+        self.validate_filter_chain(filter_chain)
+        self.task_result_filter_chains.append(filter_chain)
+
+    def _process_data_filter_chain(self, node: Node):
+        filter_chain = node.props["data"]
+        self.validate_filter_chain(filter_chain)
+        self.task_data_filter_chains.append(filter_chain)
+
+    def finalize_config(self, config_ctx: ConfigContext):
+        if self.format_version is None:
+            raise ConfigError("missing format_version")
+
+        if not isinstance(self.format_version, int):
+            raise ConfigError('"format_version" must be int, but got {}'.format(type(self.format_version)))
+
+        if self.format_version != 2:
+            raise ConfigError('wrong "format_version" {}: must be 2'.format(self.format_version))
+
+        data_filter_table = {}
+        for c in self.task_data_filter_chains:
+            assert isinstance(c, FilterChain)
+            for t in c.tasks:
+                if t in data_filter_table:
+                    raise ConfigError("multiple data filter chains defined for task {}".format(t))
+                data_filter_table[t] = c.filters
+        self.data_filter_table = data_filter_table
+
+        result_filter_table = {}
+        for c in self.task_result_filter_chains:
+            assert isinstance(c, FilterChain)
+            for t in c.tasks:
+                if t in result_filter_table:
+                    raise ConfigError("multiple data filter chains defined for task {}".format(t))
+                result_filter_table[t] = c.filters
+
+        self.result_filter_table = result_filter_table
diff --git a/nvflare/private/json_configer.py b/nvflare/private/json_configer.py
new file mode 100644
index 0000000000..79288aa250
--- /dev/null
+++ b/nvflare/private/json_configer.py
@@ -0,0 +1,118 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from nvflare.fuel.common.excepts import ConfigError
+from nvflare.fuel.utils.class_utils import ModuleScanner, get_class
+from nvflare.fuel.utils.component_builder import ComponentBuilder
+from nvflare.fuel.utils.dict_utils import extract_first_level_primitive
+from nvflare.fuel.utils.json_scanner import JsonObjectProcessor, JsonScanner, Node
+from nvflare.fuel.utils.wfconf import _EnvUpdater
+
+
+class ConfigContext(object):
+    def __init__(self):
+        self.config_json = None
+        self.pass_num = 0
+
+
+class JsonConfigurator(JsonObjectProcessor, ComponentBuilder):
+    def __init__(
+        self,
+        config_file_name: str,
+        base_pkgs: [str],
+        module_names: [str],
+        exclude_libs=True,
+        num_passes=1,
+    ):
+        JsonObjectProcessor.__init__(self)
+
+        assert isinstance(num_passes, int), "num_passes must be int"
+        assert num_passes > 0, "num_passes must > 0"
+
+        assert isinstance(config_file_name, str), "config_file_name must be str"
+        assert os.path.isfile(config_file_name), "config_file_name {} is not a valid file".format(config_file_name)
+        assert os.path.exists(config_file_name), "config_file_name {} does not exist".format(config_file_name)
+
+        self.config_file_name = config_file_name
+        self.num_passes = num_passes
+        self.module_scanner = ModuleScanner(base_pkgs, module_names, exclude_libs)
+        self.config_ctx = None
+
+        with open(config_file_name) as file:
+            self.config_data = json.load(file)
+
+        self.json_scanner = JsonScanner(self.config_data, config_file_name)
+
+    def get_module_scanner(self):
+        return self.module_scanner
+
+    def _do_configure(self):
+        config_ctx = ConfigContext()
+        config_ctx.config_json = self.config_data
+        self.config_ctx = config_ctx
+
+        all_vars = extract_first_level_primitive(self.config_data)
+        self.json_scanner.scan(_EnvUpdater(all_vars))
+
+        self.start_config(self.config_ctx)
+
+        # scan the config to create components
+        for i in range(self.num_passes):
+            self.config_ctx.pass_num = i + 1
+            self.json_scanner.scan(self)
+
+        # finalize configuration
+        self.finalize_config(self.config_ctx)
+
+    def configure(self):
+        try:
+            self._do_configure()
+        except ConfigError as ex:
+            raise ConfigError("Config error in {}: {}".format(self.config_file_name, ex))
+        except Exception as ex:
+            print("Error processing config {}: {}".format(self.config_file_name, ex))
+            raise ex
+
+    def process_element(self, node: Node):
+        self.process_config_element(self.config_ctx, node)
+
+    def is_configured_subclass(self, config_dict, base_class):
+        return issubclass(get_class(self.get_class_path(config_dict)), base_class)
+
+    def start_config(self, config_ctx: ConfigContext):
+        pass
+
+    def process_config_element(self, config_ctx: ConfigContext, node: Node):
+        pass
+
+    def finalize_config(self, config_ctx: ConfigContext):
+        pass
+
+
+def get_component_refs(component):
+    if "name" in component:
+        name = component["name"]
+        key = "name"
+    elif "path" in component:
+        name = component["path"]
+        key = "path"
+    else:
+        raise ConfigError('component has no "name" or "path')
+
+    parts = name.split("#")
+    component[key] = parts[0]
+    return parts
diff --git a/nvflare/security/__init__.py b/nvflare/security/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/security/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/security/security.py b/nvflare/security/security.py
new file mode 100644
index 0000000000..9d0e5392b9
--- /dev/null
+++ b/nvflare/security/security.py
@@ -0,0 +1,243 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.fuel.hci.server.authz import Authorizer, AuthzContext
+
+
+class Right(object):
+
+    UPLOAD_APP = "upload_app"
+    DEPLOY_ALL = "deploy_all"
+    DEPLOY_SELF = "deploy_self"
+    OPERATE_ALL = "operate_all"
+    OPERATE_SELF = "operate_self"
+    VIEW_ALL = "view_all"
+    VIEW_SELF = "view_self"
+    TRAIN_ALL = "train_all"
+    TRAIN_SELF = "train_self"
+
+
+class Rule(object):
+
+    ALLOW_BYOC = "allow_byoc"
+    ALLOW_CUSTOM_DATALIST = "allow_custom_datalist"
+
+
+class Action(object):
+
+    UPLOAD = "upload"
+    DEPLOY = "deploy"
+    OPERATE = "operate"
+    VIEW = "view"
+    TRAIN = "train"
+    BYOC = "byoc"
+    CUSTOM_DATALIST = "custom_datalist"
+
+
+ACTION_EXPLANATION = {
+    Action.UPLOAD: 'you are not authorized to upload app to "{site}"',
+    Action.DEPLOY: 'you are not authorized to deploy app to "{site}"',
+    Action.OPERATE: 'you are not authorized to operate site "{site}"',
+    Action.VIEW: 'you are not authorized to view info of site "{site}"',
+    Action.TRAIN: 'you are not authorized for training actions on site "{site}"',
+    Action.BYOC: 'the app contains custom code, which is not allowed on site "{site}"',
+    Action.CUSTOM_DATALIST: 'the app contains custom data list, which is not allowed on site "{site}"',
+}
+
+
+class FLAuthzContext(AuthzContext):
+    def __init__(self, user_name: str, site_names: [str], actions: [str]):
+        AuthzContext.__init__(self, user_name=user_name, site_names=site_names)
+        self.actions = actions
+
+    @staticmethod
+    def new_authz_context(site_names: [str], actions: [str]):
+        assert len(actions) > 0, "actions must be specified"
+        for a in actions:
+            assert a in [
+                Action.UPLOAD,
+                Action.DEPLOY,
+                Action.OPERATE,
+                Action.VIEW,
+                Action.TRAIN,
+                Action.BYOC,
+                Action.CUSTOM_DATALIST,
+            ], "invalid action {}".format(a)
+
+        return FLAuthzContext(user_name="", site_names=site_names, actions=actions)
+
+
+def action_checker_signature(user_name, site_name):
+    return True, ""
+
+
+class FLAuthorizer(Authorizer):
+    def __init__(self):
+        Authorizer.__init__(self)
+        self.action_checkers = {
+            Action.UPLOAD: self._user_can_upload,
+            Action.DEPLOY: self._user_can_deploy,
+            Action.OPERATE: self._user_can_operate,
+            Action.VIEW: self._user_can_view,
+            Action.TRAIN: self._user_can_train,
+            Action.BYOC: self._site_allows_byoc,
+            Action.CUSTOM_DATALIST: self._site_allows_custom_datalist,
+        }
+
+    def evaluate_user_right_on_site(self, right_name: str, user_name: str, site_name: str):
+        """
+        Check whether a user has a right in an org.
+        Super user has all rights in all orgs.
+        Args:
+            right_name: right to be evaluated
+            user_name: user to be evaluated against
+            site_name: the org
+
+        Returns:
+            Result:
+                True/False for bool type right;
+                Int number for int rule;
+                None if error occurred during evaluation
+            Error: error string error occurred during evaluation
+
+        """
+        right_type = self.policy.get_right_type(right_name)
+        if right_type == "bool":
+            user = self.policy.get_user(user_name)
+            if not user:
+                return None, 'unknown user "{}"'.format(user_name)
+            roles = user["roles"]
+            if "super" in roles:
+                # super user has all rights!
+                return True, ""
+
+        return super(FLAuthorizer, self).evaluate_user_right_on_site(right_name, user_name, site_name)
+
+    def _any_bool_rights(self, right_names: [str], user_name: str, site_name: str):
+        """
+        Check whether the user has any of the specified right
+        Args:
+            right_names:
+            user_name:
+            site_name:
+
+        Returns:
+
+        """
+        if not self.policy:
+            return None, "policy not defined"
+
+        user = self.policy.get_user(user_name)
+        if not user:
+            return None, 'unknown user "{}"'.format(user_name)
+
+        roles = user["roles"]
+        if "super" in roles:
+            # super user has all rights!
+            return True, ""
+
+        for right_name in right_names:
+            value, err = self.evaluate_user_right_on_site(right_name, user_name, site_name)
+            if err:
+                return None, err
+
+            if value:
+                return True, ""
+        return False, ""
+
+    def _any_bool_rules(self, rule_names: [str], site_name: str):
+        if not self.policy:
+            return None, "policy not defined"
+
+        for rule_name in rule_names:
+            value, err = self.evaluate_rule_on_site(rule_name, site_name)
+            if err:
+                return None, err
+            if value:
+                return True, ""
+        return False, ""
+
+    def _user_can_upload(self, user_name, site_name):
+        # must follow action_checker_signature
+        return self._any_bool_rights(user_name=user_name, site_name=site_name, right_names=[Right.UPLOAD_APP])
+
+    def _user_can_deploy(self, user_name, site_name):
+        # must follow action_checker_signature
+        return self._any_bool_rights(
+            user_name=user_name, site_name=site_name, right_names=[Right.DEPLOY_ALL, Right.DEPLOY_SELF]
+        )
+
+    def _user_can_train(self, user_name, site_name):
+        # must follow action_checker_signature
+        return self._any_bool_rights(
+            user_name=user_name, site_name=site_name, right_names=[Right.TRAIN_ALL, Right.TRAIN_SELF]
+        )
+
+    def _user_can_operate(self, user_name, site_name):
+        # must follow action_checker_signature
+        return self._any_bool_rights(
+            user_name=user_name, site_name=site_name, right_names=[Right.OPERATE_ALL, Right.OPERATE_SELF]
+        )
+
+    def _user_can_view(self, user_name, site_name):
+        # must follow action_checker_signature
+        return self._any_bool_rights(
+            user_name=user_name, site_name=site_name, right_names=[Right.VIEW_ALL, Right.VIEW_SELF]
+        )
+
+    def _site_allows_byoc(self, user_name, site_name):
+        # must follow action_checker_signature
+        return self._any_bool_rules(site_name=site_name, rule_names=[Rule.ALLOW_BYOC])
+
+    def _site_allows_custom_datalist(self, user_name, site_name):
+        # must follow action_checker_signature
+        return self._any_bool_rules(site_name=site_name, rule_names=[Rule.ALLOW_CUSTOM_DATALIST])
+
+    def authorize(self, ctx: FLAuthzContext):
+        if not ctx:
+            return True, ""
+
+        assert isinstance(ctx, FLAuthzContext), "authz_ctx must be FLAuthzContext but got {}".format(type(ctx))
+
+        if not ctx.actions:
+            return True, ""
+
+        assert isinstance(ctx.user_name, str) and len(ctx.user_name) > 0, "program error: no user name in ctx!"
+
+        if not ctx.site_names:
+            return True, ""
+
+        for action in ctx.actions:
+            checker = self.action_checkers.get(action, None)
+            if not checker:
+                return False, "program error: invalid action name {}".format(action)
+
+            for site_name in ctx.site_names:
+                authorized, err = checker(user_name=ctx.user_name, site_name=site_name)
+                if err:
+                    return False, err
+
+                if not authorized:
+                    text_vars = {"site": site_name, "user": ctx.user_name}
+                    return False, ACTION_EXPLANATION[action].format(**text_vars)
+
+        return True, ""
+
+
+class EmptyAuthorizer(Authorizer):
+    def authorize(self, ctx: AuthzContext) -> (object, str):
+        return True, ""
+
+    def evaluate_user_right_on_site(self, right_name: str, user_name: str, site_name: str):
+        return True, ""
diff --git a/nvflare/widgets/__init__.py b/nvflare/widgets/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/nvflare/widgets/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nvflare/widgets/comp_caller.py b/nvflare/widgets/comp_caller.py
new file mode 100644
index 0000000000..d0422e1b16
--- /dev/null
+++ b/nvflare/widgets/comp_caller.py
@@ -0,0 +1,88 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_context import FLContext
+
+from .widget import Widget
+
+
+class CallInfo(object):
+    def __init__(self, target: str, action: str, params: dict):
+        self.target = target  # target component(s) that the call is applied to
+        self.action = action  # action of the call
+        self.params = params  # params of the call
+        self.results = {}  # results of components that tried to apply the params
+
+    def report_result(self, target: str, result: str = "OK"):
+        self.results[target] = result
+
+
+class ComponentCaller(Widget):
+
+    EVENT_TYPE_CALL_COMPONENT = "comp_caller.call"
+    CTX_KEY_CALL_INFO = "comp_caller.call_info"
+
+    def __init__(self):
+        Widget.__init__(self)
+        self.engine = None
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.engine = fl_ctx.get_engine()
+        elif event_type == EventType.END_RUN:
+            self.engine = None
+
+    def call_components(self, target: str, action: str, params: dict):
+        """
+        Make a call to component(s).
+
+        Args:
+            target: the target spec of the component(s) to be called.
+            action: action of the call
+            params: parameters for the call
+
+        Returns: None or dict of result: comp name => result string
+
+        NOTE: each component that wants to participate the call mechanism must:
+
+        - Listen to the event EVENT_TYPE_CALL_COMPONENT
+
+        - In the event handler, decide whether the call is applicable to it by comparing itself to
+        the 'target'. The target could be a specific component ID, or a type of components
+
+        - decide further whether the call is applicable to it by looking at the 'action'.
+        Conceptually, the action is like a function to be called on the component.
+        If the component doesn't support the action, simply ignore the call.
+
+        - if the call is applicable, always report the execution status to the call.
+
+        """
+
+        # NOTE: it's important to assign self.engine to a new var!
+        # This is because another thread may fire the END_RUN event, which will cause
+        # self.engine to be set to None, just after checking it being None and before using it!
+        engine = self.engine
+        if not engine:
+            return None
+
+        # NOTE: we need a new context here to make sure all sticky props are copied!
+        with engine.new_context() as fl_ctx:
+            assert isinstance(fl_ctx, FLContext)
+            info = CallInfo(target=target, action=action, params=params)
+            fl_ctx.set_prop(key=self.CTX_KEY_CALL_INFO, value=info, sticky=False, private=True)
+
+            engine.fire_event(event_type=self.EVENT_TYPE_CALL_COMPONENT, fl_ctx=fl_ctx)
+
+            return info.results
diff --git a/nvflare/widgets/fed_event.py b/nvflare/widgets/fed_event.py
new file mode 100644
index 0000000000..f9946d3265
--- /dev/null
+++ b/nvflare/widgets/fed_event.py
@@ -0,0 +1,157 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+
+from nvflare.apis.client_engine_spec import ClientEngineSpec
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_constant import EventScope, FedEventHeader, FLContextKey, ReservedKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.server_engine_spec import ServerEngineSpec
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.widgets.widget import Widget
+
+FED_EVENT_TOPIC = "fed.event"
+
+
+class FedEventRunner(Widget):
+    def __init__(self, topic=FED_EVENT_TOPIC):
+        """
+        Args:
+            topic:
+        """
+        Widget.__init__(self)
+        self.topic = topic
+        self.abort_signal = None
+        self.asked_to_stop = False
+        self.engine = None
+        self.last_timestamps = {}  # client name => last_timestamp
+        self.in_events = []
+        self.in_lock = threading.Lock()
+        self.poster = threading.Thread(target=self._post, args=())
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.engine = fl_ctx.get_engine()
+            self.engine.register_aux_message_handler(topic=self.topic, message_handle_func=self._receive)
+            self.abort_signal = fl_ctx.get_run_abort_signal()
+            self.asked_to_stop = False
+            self.poster.start()
+        elif event_type == EventType.END_RUN:
+            self.asked_to_stop = True
+            if self.poster.is_alive():
+                self.poster.join()
+        else:
+            # handle outgoing fed events
+            event_scope = fl_ctx.get_prop(key=FLContextKey.EVENT_SCOPE, default=EventScope.LOCAL)
+            if event_scope != EventScope.FEDERATION:
+                return
+
+            event_data = fl_ctx.get_prop(FLContextKey.EVENT_DATA, None)
+            if not isinstance(event_data, Shareable):
+                self.log_error(fl_ctx, "bad fed event: expect data to be Shareable but got {}".format(type(event_data)))
+                return
+
+            direction = event_data.get_header(FedEventHeader.DIRECTION, "out")
+            if direction != "out":
+                # ignore incoming events
+                return
+
+            event_data.set_header(FedEventHeader.EVENT_TYPE, event_type)
+            event_data.set_header(FedEventHeader.ORIGIN, fl_ctx.get_identity_name())
+            event_data.set_header(FedEventHeader.TIMESTAMP, time.time())
+
+            targets = event_data.get_header(FedEventHeader.TARGETS, None)
+            self.fire_and_forget_request(request=event_data, fl_ctx=fl_ctx, targets=targets)
+
+    def fire_and_forget_request(self, request: Shareable, fl_ctx: FLContext, targets=None):
+        pass
+
+    def _receive(self, topic: str, request: Shareable, fl_ctx: FLContext) -> Shareable:
+        peer_name = request.get_peer_prop(ReservedKey.IDENTITY_NAME, None)
+        if not peer_name:
+            self.log_error(fl_ctx, "missing identity name of the data sender")
+            return make_reply(ReturnCode.MISSING_PEER_CONTEXT)
+
+        timestamp = request.get_header(FedEventHeader.TIMESTAMP, None)
+        if timestamp is None:
+            self.log_error(fl_ctx, "missing timestamp in incoming fed event")
+            return make_reply(ReturnCode.BAD_REQUEST_DATA)
+
+        event_type = request.get_header(FedEventHeader.EVENT_TYPE, None)
+        if event_type is None:
+            self.log_error(fl_ctx, "missing event_type in incoming fed event")
+            return make_reply(ReturnCode.BAD_REQUEST_DATA)
+
+        with self.in_lock:
+            last_timestamp = self.last_timestamps.get(peer_name, None)
+            if last_timestamp is None or timestamp > last_timestamp:
+                # we only keep new items, in case the peer somehow sent old items
+                request.set_header(FedEventHeader.DIRECTION, "in")
+                self.in_events.append(request)
+                self.last_timestamps[peer_name] = timestamp
+
+        # NOTE: we do not fire event here since event process could take time.
+        # Instead we simply add the package to the queue and return quickly.
+        # The posting of events will be handled in the poster thread
+        return make_reply(ReturnCode.OK)
+
+    def _post(self):
+        sleep_time = 0.1
+        while True:
+            time.sleep(sleep_time)
+            if self.asked_to_stop or self.abort_signal.triggered:
+                break
+
+            with self.in_lock:
+                if len(self.in_events) <= 0:
+                    continue
+                event_to_post = self.in_events.pop(0)
+                assert isinstance(event_to_post, Shareable)
+
+            if self.asked_to_stop or self.abort_signal.triggered:
+                break
+
+            with self.engine.new_context() as fl_ctx:
+                assert isinstance(fl_ctx, FLContext)
+
+                fl_ctx.set_prop(key=FLContextKey.EVENT_DATA, value=event_to_post, private=True, sticky=False)
+                fl_ctx.set_prop(key=FLContextKey.EVENT_SCOPE, value=EventScope.FEDERATION, private=True, sticky=False)
+
+                event_type = event_to_post.get_header(FedEventHeader.EVENT_TYPE)
+                self.engine.fire_event(event_type=event_type, fl_ctx=fl_ctx)
+
+
+class ServerFedEventRunner(FedEventRunner):
+    def __init__(self, topic=FED_EVENT_TOPIC):
+        FedEventRunner.__init__(self, topic)
+
+    def fire_and_forget_request(self, request: Shareable, fl_ctx: FLContext, targets=None):
+        assert isinstance(self.engine, ServerEngineSpec)
+        self.engine.fire_and_forget_aux_request(
+            topic=self.topic,
+            targets=targets,
+            request=request,
+            fl_ctx=fl_ctx,
+        )
+
+
+class ClientFedEventRunner(FedEventRunner):
+    def __init__(self, topic=FED_EVENT_TOPIC):
+        FedEventRunner.__init__(self, topic)
+
+    def fire_and_forget_request(self, request: Shareable, fl_ctx: FLContext, targets=None):
+        assert isinstance(self.engine, ClientEngineSpec)
+        self.engine.fire_and_forget_aux_request(topic=self.topic, request=request, fl_ctx=fl_ctx)
diff --git a/nvflare/widgets/info_collector.py b/nvflare/widgets/info_collector.py
new file mode 100644
index 0000000000..15b8b3a07c
--- /dev/null
+++ b/nvflare/widgets/info_collector.py
@@ -0,0 +1,186 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+
+from nvflare.apis.analytix import Data as AnalytixData
+from nvflare.apis.dxo import from_shareable
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+from .widget import Widget
+
+
+class GroupInfoCollector(object):
+    def __init__(self):
+        self.info = {}
+
+    def set_info(self, group_name: str, info: dict):
+        self.info[group_name] = info
+
+    def add_info(self, group_name: str, info: dict):
+        if group_name not in self.info:
+            self.info[group_name] = info
+        else:
+            self.info[group_name].update(info)
+
+
+class InfoCollector(Widget):
+
+    """
+
+    Info Structure:
+
+        category (dict)
+            group (dict)
+                key/value (dict)
+
+    """
+
+    CATEGORY_STATS = "stats"
+    CATEGORY_ERROR = "error"
+
+    EVENT_TYPE_GET_STATS = "info_collector.get_stats"
+    CTX_KEY_STATS_COLLECTOR = "info_collector.stats_collector"
+
+    def __init__(self):
+        Widget.__init__(self)
+        self.categories = {}
+        self.engine = None
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.reset_all()
+            self.engine = fl_ctx.get_engine()
+        elif event_type == EventType.END_RUN:
+            self.engine = None
+        elif event_type in (
+            EventType.ERROR_LOG_AVAILABLE,
+            EventType.WARNING_LOG_AVAILABLE,
+            EventType.EXCEPTION_LOG_AVAILABLE,
+        ):
+            origin = fl_ctx.get_prop(FLContextKey.EVENT_ORIGIN, None)
+            if origin:
+                group_name = str(origin)
+            else:
+                group_name = "general"
+
+            data = fl_ctx.get_prop(FLContextKey.EVENT_DATA, None)
+            if not isinstance(data, Shareable):
+                # not a valid error report
+                self.log_error(
+                    fl_ctx=fl_ctx,
+                    msg="wrong event data type for event {}: expect Shareable but got {}".format(
+                        event_type, type(data)
+                    ),
+                    fire_event=False,
+                )
+                return
+
+            try:
+                dxo = from_shareable(data)
+            except:
+                self.log_exception(
+                    fl_ctx=fl_ctx, msg="invalid event data type for event {}".format(event_type), fire_event=False
+                )
+                return
+
+            analytic_data = AnalytixData.from_dxo(dxo)
+
+            if event_type == EventType.ERROR_LOG_AVAILABLE:
+                key = "error"
+            elif event_type == EventType.WARNING_LOG_AVAILABLE:
+                key = "warning"
+            else:
+                key = "exception"
+
+            self.add_error(group_name=group_name, key=key, err=analytic_data.value)
+
+    def get_run_stats(self):
+        self.reset_category(self.CATEGORY_STATS)
+
+        # NOTE: it's important to assign self.engine to a new var!
+        # This is because another thread may fire the END_RUN event, which will cause
+        # self.engine to be set to None, just after checking it being None and before using it!
+        engine = self.engine
+        if not engine:
+            return None
+
+        # NOTE: we need a new context here to make sure all sticky props are copied!
+        # We create a new StatusCollector to hold status info.
+        # Do not use the InfoCollector itself for thread safety - multiple calls to
+        # this method (from parallel admin commands) are possible at the same time!
+        with self.engine.new_context() as fl_ctx:
+            assert isinstance(fl_ctx, FLContext)
+            coll = GroupInfoCollector()
+            fl_ctx.set_prop(key=self.CTX_KEY_STATS_COLLECTOR, value=coll, sticky=False, private=True)
+
+            engine.fire_event(event_type=self.EVENT_TYPE_GET_STATS, fl_ctx=fl_ctx)
+            # Get the StatusCollector from the fl_ctx, it could have been updated by other component.
+            coll = fl_ctx.get_prop(InfoCollector.CTX_KEY_STATS_COLLECTOR)
+            return coll.info
+
+    def add_info(self, category: str, group_name: str, key: str, value):
+        cat = self.categories.get(category, None)
+        if not cat:
+            cat = dict()
+            self.categories[category] = cat
+        group = cat.get(group_name, None)
+        if not group:
+            group = dict()
+            cat[group_name] = group
+        group[key] = value
+
+    def set_info(self, category: str, group_name: str, info: dict):
+        cat = self.categories.get(category, None)
+        if not cat:
+            cat = dict()
+            self.categories[category] = cat
+        cat[group_name] = info
+
+    def get_category(self, category: str):
+        return self.categories.get(category, None)
+
+    def get_group(self, category: str, group_name: str):
+        cat = self.categories.get(category, None)
+        if not cat:
+            return None
+        return cat.get(group_name, None)
+
+    def reset_all(self):
+        self.categories = {}
+
+    def reset_category(self, category: str):
+        self.categories[category] = {}
+
+    def reset_group(self, category: str, group_name: str):
+        cat = self.categories.get(category, None)
+        if not cat:
+            return
+        cat.get[group_name] = {}
+
+    # some convenience methods
+    def add_error(self, group_name: str, key: str, err: str):
+        now = datetime.datetime.now()
+        value = "{}: {}".format(now.strftime("%Y-%m-%d %H:%M:%S"), err)
+
+        self.add_info(category=self.CATEGORY_ERROR, group_name=group_name, key=key, value=value)
+
+    def get_errors(self):
+        return self.get_category(self.CATEGORY_ERROR)
+
+    def reset_errors(self):
+        self.reset_category(self.CATEGORY_ERROR)
diff --git a/nvflare/widgets/widget.py b/nvflare/widgets/widget.py
new file mode 100644
index 0000000000..d14dc7f199
--- /dev/null
+++ b/nvflare/widgets/widget.py
@@ -0,0 +1,43 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.fl_component import FLComponent
+
+
+class Widget(FLComponent):
+
+    """
+    A widget is used to address cross-cutting needs of all FL components.
+    Some examples of such needs:
+        - report current status
+        - dynamically change its tunable parameters
+        - record processing errors
+        - stats recording
+
+    Each widget is a singleton object that is registered with the Engine with a
+    unique ID.
+
+    All widget IDs are documented in the WidgetID class.
+
+    """
+
+    def __init__(self):
+        FLComponent.__init__(self)
+
+
+class WidgetID(object):
+
+    INFO_COLLECTOR = "info_collector"
+    COMPONENT_CALLER = "component_caller"
+    FED_EVENT_RUNNER = "fed_event_runner"
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000000..89f3775761
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,32 @@
+[tool.isort]
+known_first_party = "nvflare"
+profile = "black"
+multi_line_output = 3
+line_length = 120
+skip = [".git", ".eggs", "venv", ".venv", "versioneer.py", "_version.py", "conf.py", "nvflare/__init__.py"]
+skip_glob = [ "nvflare/private/**", "*.pyi", "**/*_pb2*" ]
+
+[tool.black]
+line-length = 120
+target-version = ['py38']
+include = '\.pyi?$'
+exclude = '''
+(
+  /(
+    # exclude a few common directories in the root of the project
+      \.eggs
+    | \.git
+    | \.hg
+    | \.mypy_cache
+    | \.tox
+    | \.venv
+    | \.pytype
+    | _build
+    | buck-out
+    | build
+    | dist
+    | nvflare/private
+  )/
+  | .pb2.
+)
+'''
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 0000000000..dbb94fc314
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,6 @@
+-r requirements-doc.txt
+isort==5.8.0
+flake8==3.9.2
+black==21.5b1
+pytest-xdist
+
diff --git a/requirements-doc.txt b/requirements-doc.txt
new file mode 100644
index 0000000000..8156e07b6d
--- /dev/null
+++ b/requirements-doc.txt
@@ -0,0 +1,5 @@
+-r requirements.txt
+sphinx>=4.1.1
+sphinx_rtd_theme
+recommonmark
+sphinxcontrib-exceltable
diff --git a/requirements-min.txt b/requirements-min.txt
new file mode 100644
index 0000000000..83535394f0
--- /dev/null
+++ b/requirements-min.txt
@@ -0,0 +1,7 @@
+PyYAML
+psutil
+numpy
+grpcio
+google-api-python-client
+cryptography
+tenseal==0.3.0
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000..8b56df2940
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+-r requirements-min.txt
diff --git a/runtest.sh b/runtest.sh
new file mode 100755
index 0000000000..e1f05728ea
--- /dev/null
+++ b/runtest.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# output formatting
+separator=""
+blue=""
+green=""
+red=""
+noColor=""
+
+if [[ -t 1 ]] # stdout is a terminal
+then
+    separator=$'--------------------------------------------------------------------------------\n'
+    blue="$(tput bold; tput setaf 4)"
+    green="$(tput bold; tput setaf 2)"
+    red="$(tput bold; tput setaf 1)"
+    noColor="$(tput sgr0)"
+fi
+
+function print_style_fail_msg() { 
+    echo "${red}Check failed!${noColor}" 
+    echo "Please run ${green}./runtest.sh${noColor} to check errors and fix them." 
+} 
+
+
+export PWD=$(pwd) && echo $PWD
+export PYTHONPATH=$PWD:$PYTHONPATH && echo $PYTHONPATH
+
+set -e
+
+# now only flake8
+python3 -m flake8 nvflare
+
+# set +e  # disable exit on failure so that diagnostics can be given on failure
+echo "${separator}${blue}isort-fix${noColor}"
+python3 -m isort --check $PWD/nvflare
+isort_status=$?
+if [ ${isort_status} -ne 0 ]
+then
+    print_style_fail_msg
+else
+    echo "${green}passed!${noColor}"
+fi
+# set -e # enable exit on failure
+
+# set +e  # disable exit on failure so that diagnostics can be given on failure
+echo "${separator}${blue}black-fix${noColor}"
+python3 -m black --check $PWD/nvflare 
+black_status=$?
+if [ ${black_status} -ne 0 ]
+then
+    print_style_fail_msg
+else
+    echo "${green}passed!${noColor}"
+fi
+# set -e
+
+pytest -q -n 24 test
+echo "Done with flake and unit test"
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000000..8a753f70a1
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,13 @@
+[flake8]
+select = B,C,E,F,N,P,T4,W,B9
+max_line_length = 120
+# C408 ignored because we like the dict keyword argument syntax
+# E501 is not flexible enough, we're using B950 instead
+ignore =
+    E203,E305,E402,E501,E721,E722,E741,F821,F841,F999,W503,W504,C408,E302,W291,E303,
+    # N812 lowercase 'torch.nn.functional' imported as non lowercase 'F'
+    N812
+per_file_ignores = __init__.py: F401
+exclude = *.pyi,.git,.eggs,nvflare/_version.py,versioneer.py,venv,.venv,_version.py,*grpc.py,*_pb2.py
+
+
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000000..46da7c0154
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,89 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+import os
+import shutil
+from datetime import datetime
+
+from setuptools import find_packages, setup
+
+nvfl_version = os.environ.get("NVFL_VERSION", "1.1.0")
+yymmdd = datetime.today().strftime("%y%m%d")
+nvfl_nightly_version = f"{nvfl_version}.dev{yymmdd}"
+
+if os.environ.get("NVFL_RELEASE"):
+    package_name = "nvflare"
+    version = nvfl_version
+else:
+    package_name = "nvflare-nightly"
+    version = nvfl_nightly_version
+
+# read the contents of your README file
+this_directory = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(this_directory, "README.md"), encoding="utf-8") as f:
+    long_description = f.read()
+
+if os.path.exists(os.path.join(this_directory, "nvflare", "poc.zip")):
+    os.remove(os.path.join(this_directory, "nvflare", "poc.zip"))
+shutil.make_archive(base_name="poc", format="zip", root_dir=os.path.join(this_directory, "nvflare"), base_dir="poc")
+shutil.move("poc.zip", os.path.join(this_directory, "nvflare", "poc.zip"))
+
+python_version = os.environ.get("PY_VERSION", "3.8")
+setup(
+    name=package_name,
+    version=version,
+    description="Federated Learning Application Runtime Environment",
+    url="https://github.com/NVIDIA/NVFlare",
+    package_dir={"nvflare": "nvflare"},
+    packages=find_packages(
+        where=".",
+        include=[
+            "*",
+        ],
+        exclude=[
+            "test",
+        ],
+    ),
+    package_data={"": ["*.yml", "*.html", "poc.zip"]},
+    zip_safe=True,
+    license_files=("LICENSE.pdf",),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: Other/Proprietary License",
+        "Operating System :: OS Independent",
+    ],
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    python_requires=">=3.8",
+    # install_requires=list(pkutils.parse_requirements("requirements.txt")),
+    install_requires=[
+        "PyYAML",
+        "psutil",
+        "numpy",
+        "grpcio",
+        "google-api-python-client",
+        "cryptography",
+        "tenseal==0.3.0",
+    ],
+    entry_points={
+        "console_scripts": [
+            "provision=nvflare.lighter.provision:main",
+            "poc=nvflare.lighter.poc:main",
+            "authz_preview=nvflare.fuel.hci.tools.authz_preview:main",
+        ],
+    },
+)
+
+os.remove(os.path.join(this_directory, "nvflare", "poc.zip"))
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000000..917ae28897
--- /dev/null
+++ b/test/__init__.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/test/app_testing/__init__.py b/test/app_testing/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/app_testing/admin_controller.py b/test/app_testing/admin_controller.py
new file mode 100644
index 0000000000..f69a3895ad
--- /dev/null
+++ b/test/app_testing/admin_controller.py
@@ -0,0 +1,160 @@
+import logging
+import time
+
+from nvflare.fuel.hci.client.api_status import APIStatus
+from nvflare.fuel.hci.client.fl_admin_api import FLAdminAPI
+from nvflare.fuel.hci.client.fl_admin_api_constants import FLDetailKey
+from nvflare.fuel.hci.client.fl_admin_api_spec import FLAdminAPIResponse, TargetType
+
+
+class AdminController(object):
+
+    def __init__(self, app_path, poll_period=10):
+        """
+        This class runs an app on a given server and clients.
+        """
+        super(AdminController, self).__init__()
+
+        self.app_path = app_path
+        self.poll_period = poll_period
+        self.last_app_name = ""
+
+        self.admin_api: FLAdminAPI = None
+        self.run_number = 0
+
+        self.logger = logging.getLogger("AdminController")
+
+    def initialize(self):
+        success = False
+        self.admin_api = FLAdminAPI(
+            host="localhost",
+            port=8003,
+            upload_dir=self.app_path,
+            download_dir=self.app_path,
+            poc=True,
+            debug=False,
+        )
+
+        try:
+            # TODO:: login or login_with_password should return FLAdminAPIResponse to be consistent?
+            response = None
+            timeout = 100
+            start_time = time.time()
+            while time.time() - start_time <= timeout:
+                response: FLAdminAPIResponse = self.admin_api.login_with_password(username="admin",
+                                                                                  password="admin")
+                if response["status"] == APIStatus.SUCCESS:
+                    success = True
+                    break
+                time.sleep(1.0)
+            if not success:
+                details = response.get('details') if response else "No details"
+                raise ValueError(f"Login to admin api failed: {details}")
+            else:
+                print("Admin successfully logged into server.")
+        except Exception as e:
+            print(f"Exception in logging in to admin: {e.__str__()}")
+
+        return success
+
+    def get_run_data(self):
+        run_data = {
+            "run_number": self.run_number,
+            "app_path": self.app_path,
+            "app_name": self.last_app_name
+        }
+
+        return run_data
+
+    def ensure_clients_started(self, num_clients):
+        if not self.admin_api:
+            return False
+
+        timeout = 1000
+        start_time = time.time()
+        clients_up = False
+        while not clients_up:
+            if time.time() - start_time > timeout:
+                raise ValueError(f"Clients could not be started in {timeout} seconds.")
+
+            response = self.admin_api.check_status(target_type=TargetType.CLIENT)
+            if response["status"] == APIStatus.SUCCESS:
+                # print(f"check client status response {response}")
+                if "details" not in response:  # clients not ready....
+                    # client response would be: {'status': <APIStatus.SUCCESS: 'SUCCESS'>, 'raw': {'time': '2021-10-29 00:09:06.220615', 'data': [{'type': 'error', 'data': 'no clients available'}], 'status': <APIStatus.SUCCESS: 'SUCCESS'>}}
+                    # How can the user know if clients are ready or not.....
+                    continue
+                for row in response["details"]["client_statuses"][1:]:
+                    if row[3] != "not started":
+                        continue
+                # wait for all clients to come up
+                if len(response["details"]["client_statuses"]) < num_clients + 1:
+                    continue
+                clients_up = True
+                print("All clients are up.")
+            time.sleep(1.0)
+
+        return clients_up
+
+    def server_status(self):
+        if not self.admin_api:
+            return ""
+
+        response = self.admin_api.check_status(target_type=TargetType.SERVER)
+        if response["status"] == APIStatus.SUCCESS:
+            if "details" in response:
+                return response["details"]
+        return ""
+
+    def client_status(self):
+        if not self.admin_api:
+            return ""
+
+        response = self.admin_api.check_status(target_type=TargetType.CLIENT)
+        if response["status"] == APIStatus.SUCCESS:
+            if "details" in response:
+                return response["details"]
+        return ""
+
+    def deploy_app(self, app_name) -> bool:
+        if not self.admin_api:
+            return False
+
+        self.run_number += 1
+
+        response = self.admin_api.set_run_number(self.run_number)
+        if response["status"] != APIStatus.SUCCESS:
+            raise RuntimeError(f"set run number failed: {response}")
+        response = self.admin_api.upload_app(app_name)
+        if response["status"] != APIStatus.SUCCESS:
+            raise RuntimeError(f"upload_app failed: {response}")
+        response = self.admin_api.deploy_app(app=app_name, target_type=TargetType.ALL)
+        if response["status"] != APIStatus.SUCCESS:
+            raise RuntimeError(f"deploy_app failed: {response}")
+        response = self.admin_api.start_app(target_type=TargetType.ALL)
+        if response["status"] != APIStatus.SUCCESS:
+            raise RuntimeError(f"start_app failed: {response}")
+
+        self.last_app_name = app_name
+
+        return True
+
+    def run_app(self):
+        #! Is it possible to get the training log after training is done?
+        training_done = False
+        while not training_done:
+            response = self.admin_api.check_status(target_type=TargetType.SERVER)
+            if response["status"] != APIStatus.SUCCESS:
+                raise RuntimeError(f"check_status failed: {response}")
+            if response["details"][FLDetailKey.SERVER_ENGINE_STATUS] == "stopped":
+                response = self.admin_api.check_status(target_type=TargetType.CLIENT)
+                if response["status"] != APIStatus.SUCCESS:
+                    raise RuntimeError(f"check_status failed: {response}")
+                for row in response["details"]["client_statuses"]:
+                    if row[3] != "stopped":
+                        continue
+                training_done = True
+            time.sleep(self.poll_period)
+
+    def finalize(self):
+        self.admin_api.shutdown(target_type=TargetType.ALL)
diff --git a/test/app_testing/app_result_validator.py b/test/app_testing/app_result_validator.py
new file mode 100644
index 0000000000..62927017ac
--- /dev/null
+++ b/test/app_testing/app_result_validator.py
@@ -0,0 +1,12 @@
+import logging
+
+
+class AppResultValidator(object):
+
+    def __init__(self):
+        super(AppResultValidator, self).__init__()
+        self.logger = logging.getLogger("AppValidator")
+
+    def validate_results(self, server_data, client_data, run_data) -> bool:
+        pass
+
diff --git a/test/app_testing/np/__init__.py b/test/app_testing/np/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/app_testing/np/constants.py b/test/app_testing/np/constants.py
new file mode 100755
index 0000000000..7e0e5dc276
--- /dev/null
+++ b/test/app_testing/np/constants.py
@@ -0,0 +1,2 @@
+class NPConstants:
+    NUMPY_KEY = "numpy_key"
diff --git a/test/app_testing/np/np_formatter.py b/test/app_testing/np/np_formatter.py
new file mode 100755
index 0000000000..9e8b6fcbc4
--- /dev/null
+++ b/test/app_testing/np/np_formatter.py
@@ -0,0 +1,48 @@
+from nvflare.apis.dxo import DataKind, from_bytes
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.formatter import Formatter
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPFormatter(Formatter):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def format(self, fl_ctx: FLContext) -> str:
+        """The format function gets validation shareable locations from the dictionary. It loads each shareable,
+        get the validation results and converts it into human readable string.
+
+        Args:
+            fl_ctx (FLContext): FLContext object.
+
+        Returns:
+            str: Human readable validation results.
+        """
+        # Get the val shareables
+        validation_shareables_dict = fl_ctx.get_prop(AppConstants.VALIDATION_RESULT, {})
+
+        # Result dictionary
+        res = {}
+
+        try:
+            # This is a 2d dictionary with each validation result at
+            # validation_shareables_dict[data_client][model_client]
+            for data_client in validation_shareables_dict.keys():
+                validation_dict = validation_shareables_dict[data_client]
+                if validation_dict:
+                    res[data_client] = {}
+                    for model_name in validation_dict.keys():
+                        dxo_path = validation_dict[model_name]
+
+                        # Load the shareable
+                        with open(dxo_path, "rb") as f:
+                            metric_dxo = from_bytes(f.read())
+
+                        # Get metrics from shareable
+                        if metric_dxo and metric_dxo.data_kind == DataKind.METRICS:
+                            metrics = metric_dxo.data
+                            res[data_client][model_name] = metrics
+        except Exception as e:
+            self.log_error(fl_ctx, f"Exception: {e.__str__()}")
+
+        return f"{res}"
diff --git a/test/app_testing/np/np_model_locator.py b/test/app_testing/np/np_model_locator.py
new file mode 100755
index 0000000000..ee0215af70
--- /dev/null
+++ b/test/app_testing/np/np_model_locator.py
@@ -0,0 +1,62 @@
+import os
+from typing import List
+
+import numpy as np
+
+from nvflare.apis.dxo import DXO, DataKind
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model_locator import ModelLocator
+
+from .constants import NPConstants
+
+
+class NPModelLocator(ModelLocator):
+    SERVER_MODEL_NAME = "server"
+
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        """The ModelLocator's job is to find the models to be included for cross site evaluation
+        located on server. This NPModelLocator finds and extracts "server" model that is saved during training.
+
+        Args:
+            model_dir (str): Directory to look for models in. Defaults to "model"
+            model_name (str). Name of the model. Defaults to "server.npy"
+        """
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_file_name = model_name
+
+    def get_model_names(self, fl_ctx: FLContext) -> List[str]:
+        """Returns the list of model names that should be included from server in cross site validation.add()
+
+        Args:
+            fl_ctx (FLContext): FL Context object.
+
+        Returns:
+            List[str]: List of model names.
+        """
+        return [NPModelLocator.SERVER_MODEL_NAME]
+
+    def locate_model(self, model_name, fl_ctx: FLContext) -> DXO:
+        dxo = None
+        engine = fl_ctx.get_engine()
+
+        if model_name == NPModelLocator.SERVER_MODEL_NAME:
+            run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+            run_dir = engine.get_workspace().get_run_dir(run_number)
+            model_path = os.path.join(run_dir, self.model_dir)
+
+            model_load_path = os.path.join(model_path, self.model_file_name)
+            np_data = None
+            try:
+                np_data = np.load(model_load_path, allow_pickle=True)
+                self.log_info(fl_ctx, f"Loaded {model_name} model from {model_load_path}.")
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load NP Model: {e}.")
+
+            if np_data is not None:
+                weights = {NPConstants.NUMPY_KEY: np_data}
+                dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights, meta={})
+
+        return dxo
diff --git a/test/app_testing/np/np_model_persistor.py b/test/app_testing/np/np_model_persistor.py
new file mode 100755
index 0000000000..00ea270069
--- /dev/null
+++ b/test/app_testing/np/np_model_persistor.py
@@ -0,0 +1,61 @@
+import os
+
+import numpy as np
+
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable, ModelLearnableKey, make_model_learnable
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from nvflare.app_common.app_constant import AppConstants
+
+from .constants import NPConstants
+
+
+class NPModelPersistor(ModelPersistor):
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_name = model_name
+
+        # This is default model that will be used if not local model is provided.
+        self.default_data = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32)
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        # Get start round from FLContext. If start_round > 0, we will try loading model from disk.
+        start_round = fl_ctx.get_prop(AppConstants.START_ROUND, 0)
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir, self.model_name)
+
+        # Create a new numpy model
+        if start_round > 0:
+            try:
+                data = np.load(model_path)
+            except Exception as e:
+                self.log_info(fl_ctx, f"Unable to load model from {model_path}. Using default data instead.")
+                data = self.default_data.copy()
+        else:
+            data = self.default_data.copy()
+
+        # Generate model dictionary and create model_learnable.
+        weights = {NPConstants.NUMPY_KEY: data}
+        model_learnable = make_model_learnable(weights, {})
+
+        self.logger.info(f"Loaded initial model: {model_learnable[ModelLearnableKey.WEIGHTS]}")
+        return model_learnable
+
+    def save_model(self, model: ModelLearnable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self.model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
diff --git a/test/app_testing/np/np_trainer.py b/test/app_testing/np/np_trainer.py
new file mode 100755
index 0000000000..ea2b1dee0e
--- /dev/null
+++ b/test/app_testing/np/np_trainer.py
@@ -0,0 +1,203 @@
+import os
+import time
+
+import numpy as np
+
+from nvflare.apis.dxo import DXO, DataKind, from_shareable
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.app_constant import AppConstants
+
+from .constants import NPConstants
+
+
+class NPTrainer(Executor):
+    def __init__(
+        self,
+        delta=1,
+        sleep_time=0,
+        train_task_name=AppConstants.TASK_TRAIN,
+        submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL,
+        model_name="best_numpy.npy",
+        model_dir="model",
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        if not (isinstance(delta, float) or isinstance(delta, int)):
+            raise TypeError("delta must be an instance of float or int.")
+
+        self._delta = delta
+        self._model_name = model_name
+        self._model_dir = model_dir
+        self._sleep_time = sleep_time
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     Create all major components here. This is a simple app that doesn't need any components.
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any long tasks should check abort_signal regularly. Otherwise abort client
+        # will not work.
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._train_task_name:
+            # First we extract DXO from the shareable.
+            try:
+                incoming_dxo = from_shareable(shareable)
+            except BaseException as e:
+                self.system_panic(f"Unable to convert shareable to model definition. Exception {e.__str__()}", fl_ctx)
+                return self._get_exception_shareable()
+
+            # Information about workflow is retreived from the shareable header.
+            current_round = shareable.get_header(AppConstants.CURRENT_ROUND, None)
+            total_rounds = shareable.get_header(AppConstants.NUM_ROUNDS, None)
+
+            # Ensure that data is of type weights. Extract model data.
+            if incoming_dxo.data_kind != DataKind.WEIGHTS:
+                self.system_panic("Model dex should be of kind DataKind.WEIGHTS.", fl_ctx)
+                return self._get_exception_shareable()
+            np_data = incoming_dxo.data
+
+            # Display properties.
+            self.log_info(fl_ctx, f"Incoming data kind: {incoming_dxo.data_kind}")
+            self.log_info(fl_ctx, f"Model: \n{np_data}")
+            self.log_info(fl_ctx, f"Current Round: {current_round}")
+            self.log_info(fl_ctx, f"Total Rounds: {total_rounds}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Doing some dummy training.
+            if np_data:
+                if NPConstants.NUMPY_KEY in np_data:
+                    np_data[NPConstants.NUMPY_KEY] += self._delta
+                else:
+                    self.log_error(fl_ctx, "numpy_key not found in model.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+                    return shareable
+            else:
+                self.log_error(fl_ctx, "No model weights found in shareable.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # We check abort_signal regularly to make sure
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Save local numpy model
+            try:
+                self._save_local_model(fl_ctx, np_data)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Exception in saving local model: {e}.")
+
+            self.log_info(
+                fl_ctx,
+                f"Model after training: {np_data}",
+            )
+
+            # Checking abort signal again.
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Prepare a DXO for our updated model. Create shareable and return
+            outgoing_dxo = DXO(data_kind=incoming_dxo.data_kind, data=np_data, meta={})
+            return outgoing_dxo.to_shareable()
+        elif task_name == self._submit_model_task_name:
+            # Retrieve the local model saved during training.
+            np_data = None
+            try:
+                np_data = self._load_local_model(fl_ctx)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load model: {e}")
+
+            # Checking abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Create DXO and shareable from model data.
+            model_shareable = Shareable()
+            if np_data:
+                outgoing_dxo = DXO(data_kind=DataKind.WEIGHTS, data=np_data)
+                model_shareable = outgoing_dxo.to_shareable()
+            else:
+                # Set return code.
+                self.log_error(fl_ctx, f"local model not found.")
+                model_shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+
+            return model_shareable
+        else:
+            # If unknown task name, set RC accordingly.
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+            return shareable
+
+    def _load_local_model(self, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+
+        model_load_path = os.path.join(model_path, self._model_name)
+        np_data = None
+        try:
+            np_data = np.load(model_load_path)
+        except Exception as e:
+            self.log_error(fl_ctx, f"Unable to load local model: {e.__str__()}")
+            return None
+
+        model = ModelLearnable()
+        model[NPConstants.NUMPY_KEY] = np_data
+
+        return model
+
+    def _save_local_model(self, fl_ctx: FLContext, model: dict):
+        # Save local model
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self._model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
+
+    def _get_exception_shareable(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/np/np_validator.py b/test/app_testing/np/np_validator.py
new file mode 100755
index 0000000000..e23165d5f7
--- /dev/null
+++ b/test/app_testing/np/np_validator.py
@@ -0,0 +1,127 @@
+import logging
+import time
+
+import numpy as np
+
+from nvflare.apis.dxo import DXO, DataKind, from_shareable
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.app_constant import AppConstants
+
+from .constants import NPConstants
+
+
+class NPValidator(Executor):
+    def __init__(
+        self,
+        epsilon=1,
+        sleep_time=0,
+        validate_task_name=AppConstants.TASK_VALIDATION,
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        self.logger = logging.getLogger("NPValidator")
+        self._random_epsilon = epsilon
+        self._sleep_time = sleep_time
+        self._validate_task_name = validate_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     Create all major components here. This is a simple app that doesn't need any components.
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any long tasks should check abort_signal regularly. Otherwise abort client
+        # will not work.
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._abort_execution()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._validate_task_name:
+            # First we extract DXO from the shareable.
+            try:
+                model_dxo = from_shareable(shareable)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to extract model dxo from shareable. Exception: {e.__str__()}")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # Get model from shareable. data_kind must be WEIGHTS.
+            if model_dxo.data and model_dxo.data_kind == DataKind.WEIGHTS:
+                model = model_dxo.data
+            else:
+                self.log_error(
+                    fl_ctx, f"Model Dex doesn't have data or is not of type DataKind.WEIGHTS. Unable  " "to validate."
+                )
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # The workflow provides MODEL_OWNER information in the shareable header.
+            model_name = shareable.get_header(AppConstants.MODEL_OWNER, "?")
+
+            # Print properties.
+            self.log_info(fl_ctx, f"Model: \n{model}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+            self.log_info(fl_ctx, f"Validating model from {model_name}.")
+
+            # Check abort signal regularly.
+            if abort_signal.triggered:
+                return self._abort_execution()
+
+            # Check if key exists in model
+            if NPConstants.NUMPY_KEY not in model:
+                self.log_error(fl_ctx, "numpy_key not in model. Unable to validate.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # Do some dummy validation.
+            random_epsilon = np.random.random()
+            self.log_info(fl_ctx, f"Adding random epsilon {random_epsilon} in validation.")
+            val_results = {}
+            np_data = model[NPConstants.NUMPY_KEY]
+            np_data = np.sum(np_data / np.max(np_data))
+            val_results["accuracy"] = np_data + random_epsilon
+
+            # Check abort signal regularly.
+            if abort_signal.triggered:
+                return self._abort_execution()
+
+            self.log_info(fl_ctx, f"Validation result: {val_results}")
+
+            # Create DXO for metrics and return shareable.
+            metric_dxo = DXO(data_kind=DataKind.METRICS, data=val_results)
+            return metric_dxo.to_shareable()
+
+        else:
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+            return shareable
+
+    def _abort_execution(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/site_launcher.py b/test/app_testing/site_launcher.py
new file mode 100644
index 0000000000..674bbf218f
--- /dev/null
+++ b/test/app_testing/site_launcher.py
@@ -0,0 +1,201 @@
+import logging
+import os
+import shlex
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+import traceback
+
+
+def process_logs(log_path, pid):
+    try:
+        with open(log_path, 'ab') as file:
+            for line in pid.stdout:  # b'\n'-separated lines
+                sys.stdout.buffer.write(line)  # pass bytes as is
+                file.write(line)
+                file.flush()
+
+    except BaseException as e:
+        traceback.print_exc()
+        print(f"Exception in process_logs for file {log_path}: {e.__str__()}")
+
+
+class SiteLauncher(object):
+
+    def __init__(self, poc_directory, server_dir_name="server", client_dir_name="client", admin_dir_name="admin"
+                 , app_path=None):
+        """
+        This class sets up the test environment for a test. It will launch and keep track of servers and clients.
+        """
+        super(SiteLauncher, self).__init__()
+
+        self.original_poc_directory = poc_directory
+        self.server_dir_name = server_dir_name
+        self.client_dir_name = client_dir_name
+        self.admin_dir_name = admin_dir_name
+        self.app_path = app_path
+
+        self.server_properties = {}
+        self.client_properties = {}
+
+        self.admin_api = None
+
+        self.logger = logging.getLogger("SiteRunner")
+
+        # Create temporary poc directory
+        if not os.path.exists(self.original_poc_directory):
+            raise RuntimeError("Please run POC command first and provide the POC path!")
+
+        # TODO: What is log directory and should it be added here?
+        root_dir = tempfile.mkdtemp()
+        shutil.copytree(self.original_poc_directory, root_dir, dirs_exist_ok=True)
+        self.poc_directory = root_dir
+        print(f"Using root dir: {root_dir}")
+
+    def start_server(self):
+        server_dir = os.path.join(self.poc_directory, self.server_dir_name)
+        log_path = os.path.join(server_dir, 'log.txt')
+        # log_file = open(log_path, 'w')
+        new_env = os.environ.copy()
+
+        # Create upload directory
+        os.makedirs(os.path.join(server_dir, "transfer"), exist_ok=True)
+
+        command = f"{sys.executable} -m nvflare.private.fed.app.server.server_train " \
+                  f"-m {server_dir} -s fed_server.json" \
+                  f" --set secure_train=false config_folder=config"
+        process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=new_env,
+                                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        # process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=new_env)
+        print("Starting server ...")
+
+        t = threading.Thread(target=process_logs, args=(log_path, process))
+        t.start()
+
+        self.server_properties["path"] = server_dir
+        self.server_properties["process"] = process
+        # self.server_properties["log_file"] = log_file
+        self.server_properties["log_path"] = log_path
+
+    def start_clients(self, n=2):
+        # Make sure that previous clients are killed
+        self.stop_all_clients()
+        self.client_properties.clear()
+
+        # For each client, copy the directory
+        src_client_directory = os.path.join(self.poc_directory, self.client_dir_name)
+        new_env = os.environ.copy()
+
+        for i in range(n):
+            client_id = i
+            client_name = self.client_dir_name + f"_{client_id}"
+
+            # Copy and create new directory
+            client_dir_name = os.path.join(self.poc_directory, client_name)
+            shutil.copytree(src_client_directory, client_dir_name)
+            log_path = os.path.join(client_dir_name, 'log.txt')
+
+            self.client_properties[client_id] = {}
+            self.client_properties[client_id]["path"] = client_dir_name
+            self.client_properties[client_id]["name"] = client_name
+            self.client_properties[client_id]["log_path"] = log_path
+
+            # Launch the new client
+            client_startup_dir = os.path.join(client_dir_name)
+            command = f"{sys.executable} -m nvflare.private.fed.app.client.client_train -m " \
+                      f"{client_startup_dir} -s fed_client.json --set secure_train=false config_folder=config" \
+                      f" uid=client_{client_id}"
+
+            process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=new_env,
+                                       stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            # process = subprocess.Popen(shlex.split(command, " "), preexec_fn=os.setsid, env=new_env)
+            self.client_properties[client_id]["process"] = process
+
+            print(f"Launched client {client_id} process.")
+
+            t = threading.Thread(target=process_logs, args=(log_path, process))
+            t.start()
+
+    def get_server_data(self):
+        if "log_file" in self.server_properties:
+            self.server_properties["log_file"].flush()
+
+        server_data = {
+            "server_path": self.server_properties["path"],
+            "server_process": self.server_properties["process"],
+            "server_name": self.server_dir_name,
+            "root_dir": self.poc_directory,
+            "log_path": self.server_properties["log_path"]
+        }
+
+        return server_data
+
+    def get_client_data(self):
+        client_data = {
+            "client_paths": [self.client_properties[x]["path"] for x in self.client_properties],
+            "client_names": [self.client_properties[x]["name"] for x in self.client_properties],
+            "client_processes": [self.client_properties[x]["process"] for x in self.client_properties]
+        }
+
+        return client_data
+
+    def stop_server(self):
+        # Kill all clients first
+        try:
+            self.stop_all_clients()
+
+            # Kill the process
+            if "process" in self.server_properties and self.server_properties["process"]:
+                os.killpg(self.server_properties["process"].pid, signal.SIGTERM)
+
+                subprocess.call(['kill', str(self.server_properties["process"].pid)])
+                self.server_properties["process"].wait()
+                print("Sent SIGTERM to server.")
+            else:
+                print("No server process.")
+        except Exception as e:
+            print(f"Exception in stopping server: {e.__str__()}")
+        finally:
+            self.server_properties.clear()
+
+    def stop_client(self, client_id) -> bool:
+        if client_id not in self.client_properties:
+            print(f"Client {client_id} not present in client processes.")
+            return False
+        if not self.client_properties[client_id]["process"]:
+            print(f"Client {client_id} process is None.")
+            self.client_properties.pop(client_id)
+            return False
+
+        try:
+            os.killpg(self.client_properties[client_id]["process"].pid, signal.SIGTERM)
+
+            subprocess.call(['kill', str(self.client_properties[client_id]["process"].pid)])
+            self.client_properties[client_id]["process"].wait()
+
+            self.client_properties.pop(client_id)
+
+            print(f"Sent SIGTERM to client {client_id}.")
+        except Exception as e:
+            print(f"Exception in stopping client {client_id}: {e.__str__()}")
+            return False
+
+        return True
+
+    def stop_all_clients(self):
+        client_ids = list(self.client_properties.keys())
+        for client_id in client_ids:
+            self.stop_client(client_id)
+
+    def stop_all_sites(self):
+        self.stop_server()
+
+    def client_status(self, client_id):
+        pass
+
+    def finalize(self):
+        print(f"Deleting temporary directory: {self.poc_directory}.")
+        shutil.rmtree(self.poc_directory)
diff --git a/test/app_testing/test_apps/cross_val/config/config_fed_client.json b/test/app_testing/test_apps/cross_val/config/config_fed_client.json
new file mode 100755
index 0000000000..840d10aec0
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val/config/config_fed_client.json
@@ -0,0 +1,27 @@
+{
+  "format_version": 2,
+  "handlers": [],
+  "executors": [
+    {
+      "tasks": [
+        "train",
+        "submit_model"
+      ],
+      "executor": {
+        "path": "np_trainer.NPTrainer",
+        "args": {}
+      }
+    },
+    {
+      "tasks": [
+        "validate"
+      ],
+      "executor": {
+        "path": "np_validator.NPValidator"
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/cross_val/config/config_fed_server.json b/test/app_testing/test_apps/cross_val/config/config_fed_server.json
new file mode 100755
index 0000000000..7bf9f5270e
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val/config/config_fed_server.json
@@ -0,0 +1,65 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "handlers": [],
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    },
+    {
+      "id": "model_locator",
+      "path": "np_model_locator.NPModelLocator",
+      "args": {}
+    },
+    {
+      "id": "formatter",
+      "path": "np_formatter.NPFormatter",
+      "args": {}
+    }
+  ],
+  "workflows": [
+    {
+      "id": "scatter_and_gather",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 6000
+      }
+    },
+    {
+      "id": "cross_site_model_eval",
+      "path": "nvflare.app_common.workflows.cross_site_model_eval.CrossSiteModelEval",
+      "args": {
+        "model_locator_id": "model_locator",
+        "submit_model_timeout": 600,
+        "validation_timeout": 6000,
+        "cleanup_models": false
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/cross_val/custom/__init__.py b/test/app_testing/test_apps/cross_val/custom/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/app_testing/test_apps/cross_val/custom/constants.py b/test/app_testing/test_apps/cross_val/custom/constants.py
new file mode 100755
index 0000000000..7e0e5dc276
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val/custom/constants.py
@@ -0,0 +1,2 @@
+class NPConstants:
+    NUMPY_KEY = "numpy_key"
diff --git a/test/app_testing/test_apps/cross_val/custom/np_formatter.py b/test/app_testing/test_apps/cross_val/custom/np_formatter.py
new file mode 100755
index 0000000000..74fa0f966f
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val/custom/np_formatter.py
@@ -0,0 +1,51 @@
+from nvflare.apis.dxo import from_bytes, DataKind
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.formatter import \
+    Formatter
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPFormatter(Formatter):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def format(self, fl_ctx: FLContext) -> str:
+        """The format function gets validation shareable locations from the dictionary. It loads each shareable,
+        get the validation results and converts it into human readable string.
+
+        Args:
+            fl_ctx (FLContext): FLContext object.
+
+        Returns:
+            str: Human readable validation results.
+        """
+        # Get the val shareables
+        validation_shareables_dict = fl_ctx.get_prop(
+            AppConstants.VALIDATION_RESULT, {}
+        )
+
+        # Result dictionary
+        res = {}
+
+        try:
+            # This is a 2d dictionary with each validation result at
+            # validation_shareables_dict[data_client][model_client]
+            for data_client in validation_shareables_dict.keys():
+                validation_dict = validation_shareables_dict[data_client]
+                if validation_dict:
+                    res[data_client] = {}
+                    for model_name in validation_dict.keys():
+                        dxo_path = validation_dict[model_name]
+
+                        # Load the shareable
+                        with open(dxo_path, "rb") as f:
+                            metric_dxo = from_bytes(f.read())
+
+                        # Get metrics from shareable
+                        if metric_dxo and metric_dxo.data_kind== DataKind.METRICS:
+                            metrics = metric_dxo.data
+                            res[data_client][model_name] = metrics
+        except Exception as e:
+            self.log_error(fl_ctx, f"Exception: {e.__str__()}")
+
+        return f"{res}"
diff --git a/test/app_testing/test_apps/cross_val/custom/np_model_locator.py b/test/app_testing/test_apps/cross_val/custom/np_model_locator.py
new file mode 100755
index 0000000000..a36f8e99c1
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val/custom/np_model_locator.py
@@ -0,0 +1,63 @@
+import os
+from typing import List
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, DataKind
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model_locator import (
+    ModelLocator,
+)
+
+
+class NPModelLocator(ModelLocator):
+    SERVER_MODEL_NAME = "server"
+
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        """The ModelLocator's job is to find the models to be included for cross site evaluation
+        located on server. This NPModelLocator finds and extracts "server" model that is saved during training.
+
+        Args:
+            model_dir (str): Directory to look for models in. Defaults to "model"
+            model_name (str). Name of the model. Defaults to "server.npy"
+        """
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_file_name = model_name
+
+    def get_model_names(self, fl_ctx: FLContext) -> List[str]:
+        """Returns the list of model names that should be included from server in cross site validation.add()
+
+        Args:
+            fl_ctx (FLContext): FL Context object.
+
+        Returns:
+            List[str]: List of model names.
+        """
+        return [NPModelLocator.SERVER_MODEL_NAME]
+
+    def locate_model(self, model_name, fl_ctx: FLContext) -> DXO:
+        dxo = None
+        engine = fl_ctx.get_engine()
+
+        if model_name == NPModelLocator.SERVER_MODEL_NAME:
+            run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+            run_dir = engine.get_workspace().get_run_dir(run_number)
+            model_path = os.path.join(run_dir, self.model_dir)
+
+            model_load_path = os.path.join(model_path, self.model_file_name)
+            np_data = None
+            try:
+                np_data = np.load(model_load_path, allow_pickle=True)
+                self.log_info(fl_ctx, f"Loaded {model_name} model from {model_load_path}.")
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load NP Model: {e}.")
+
+            if np_data is not None:
+                weights = {NPConstants.NUMPY_KEY: np_data}
+                dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights, meta={})
+
+        return dxo
diff --git a/test/app_testing/test_apps/cross_val/custom/np_model_persistor.py b/test/app_testing/test_apps/cross_val/custom/np_model_persistor.py
new file mode 100755
index 0000000000..7818abd30a
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val/custom/np_model_persistor.py
@@ -0,0 +1,65 @@
+import os
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable, make_model_learnable, ModelLearnableKey
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPModelPersistor(ModelPersistor):
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_name = model_name
+
+        # This is default model that will be used if not local model is provided.
+        self.default_data = np.array(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32
+        )
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        # Get start round from FLContext. If start_round > 0, we will try loading model from disk.
+        start_round = fl_ctx.get_prop(AppConstants.START_ROUND, 0)
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir, self.model_name)
+
+        # Create a new numpy model
+        if start_round > 0:
+            try:
+                data = np.load(model_path)
+            except Exception as e:
+                self.log_info(
+                    fl_ctx,
+                    f"Unable to load model from {model_path}. Using default data instead."
+                )
+                data = self.default_data.copy()
+        else:
+            data = self.default_data.copy()
+
+        # Generate model dictionary and create model_learnable.
+        weights = {NPConstants.NUMPY_KEY: data}
+        model_learnable = make_model_learnable(weights, {})
+
+        self.logger.info(f"Loaded initial model: {model_learnable[ModelLearnableKey.WEIGHTS]}")
+        return model_learnable
+
+    def save_model(self, model: ModelLearnable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self.model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
diff --git a/test/app_testing/test_apps/cross_val/custom/np_trainer.py b/test/app_testing/test_apps/cross_val/custom/np_trainer.py
new file mode 100755
index 0000000000..6ef031001d
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val/custom/np_trainer.py
@@ -0,0 +1,202 @@
+import os
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, from_shareable, DataKind
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPTrainer(Executor):
+    def __init__(
+        self,
+        delta=1,
+        sleep_time=0,
+        train_task_name=AppConstants.TASK_TRAIN,
+        submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL,
+        model_name="best_numpy.npy",
+        model_dir="model",
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        if not (isinstance(delta, float) or isinstance(delta, int)):
+            raise TypeError("delta must be an instance of float or int.")
+
+        self._delta = delta
+        self._model_name = model_name
+        self._model_dir = model_dir
+        self._sleep_time = sleep_time
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     Create all major components here. This is a simple app that doesn't need any components.
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any long tasks should check abort_signal regularly. Otherwise abort client
+        # will not work.
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._train_task_name:
+            # First we extract DXO from the shareable.
+            try:
+                incoming_dxo = from_shareable(shareable)
+            except BaseException as e:
+                self.system_panic(f"Unable to convert shareable to model definition. Exception {e.__str__()}", fl_ctx)
+                return self._get_exception_shareable()
+
+            # Information about workflow is retreived from the shareable header.
+            current_round = shareable.get_header(AppConstants.CURRENT_ROUND, None)
+            total_rounds = shareable.get_header(AppConstants.NUM_ROUNDS, None)
+
+            # Ensure that data is of type weights. Extract model data.
+            if incoming_dxo.data_kind != DataKind.WEIGHTS:
+                self.system_panic("Model dex should be of kind DataKind.WEIGHTS.", fl_ctx)
+                return self._get_exception_shareable()
+            np_data = incoming_dxo.data
+
+            # Display properties.
+            self.log_info(fl_ctx, f"Incoming data kind: {incoming_dxo.data_kind}")
+            self.log_info(fl_ctx, f"Model: \n{np_data}")
+            self.log_info(fl_ctx, f"Current Round: {current_round}")
+            self.log_info(fl_ctx, f"Total Rounds: {total_rounds}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Doing some dummy training.
+            if np_data:
+                if NPConstants.NUMPY_KEY in np_data:
+                    np_data[NPConstants.NUMPY_KEY] += self._delta
+                else:
+                    self.log_error(fl_ctx, "numpy_key not found in model.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+                    return shareable
+            else:
+                self.log_error(fl_ctx, "No model weights found in shareable.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # We check abort_signal regularly to make sure
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Save local numpy model
+            try:
+                self._save_local_model(fl_ctx, np_data)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Exception in saving local model: {e}.")
+
+            self.log_info(
+                fl_ctx,
+                f"Model after training: {np_data}",
+            )
+
+            # Checking abort signal again.
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Prepare a DXO for our updated model. Create shareable and return
+            outgoing_dxo = DXO(data_kind=incoming_dxo.data_kind, data=np_data, meta={})
+            return outgoing_dxo.to_shareable()
+        elif task_name == self._submit_model_task_name:
+            # Retrieve the local model saved during training.
+            np_data = None
+            try:
+                np_data = self._load_local_model(fl_ctx)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load model: {e}")
+
+            # Checking abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Create DXO and shareable from model data.
+            model_shareable = Shareable()
+            if np_data:
+                outgoing_dxo = DXO(data_kind=DataKind.WEIGHTS, data=np_data)
+                model_shareable = outgoing_dxo.to_shareable()
+            else:
+                # Set return code.
+                self.log_error(fl_ctx, f"local model not found.")
+                model_shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+
+            return model_shareable
+        else:
+            # If unknown task name, set RC accordingly.
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+            return shareable
+
+    def _load_local_model(self, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+
+        model_load_path = os.path.join(model_path, self._model_name)
+        np_data = None
+        try:
+            np_data = np.load(model_load_path)
+        except Exception as e:
+            self.log_error(fl_ctx, f"Unable to load local model: {e.__str__()}")
+            return None
+
+        model = ModelLearnable()
+        model[NPConstants.NUMPY_KEY] = np_data
+
+        return model
+
+    def _save_local_model(self, fl_ctx: FLContext, model: dict):
+        # Save local model
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self._model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
+
+    def _get_exception_shareable(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/test_apps/cross_val/custom/np_validator.py b/test/app_testing/test_apps/cross_val/custom/np_validator.py
new file mode 100755
index 0000000000..519ca55010
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val/custom/np_validator.py
@@ -0,0 +1,127 @@
+import logging
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import from_shareable, DataKind, DXO
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPValidator(Executor):
+    def __init__(
+        self,
+        epsilon=1,
+        sleep_time=0,
+        validate_task_name=AppConstants.TASK_VALIDATION,
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        self.logger = logging.getLogger("NPValidator")
+        self._random_epsilon = epsilon
+        self._sleep_time = sleep_time
+        self._validate_task_name = validate_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     Create all major components here. This is a simple app that doesn't need any components.
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any long tasks should check abort_signal regularly. Otherwise abort client
+        # will not work.
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._abort_execution()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._validate_task_name:
+            # First we extract DXO from the shareable.
+            try:
+                model_dxo = from_shareable(shareable)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to extract model dxo from shareable. Exception: {e.__str__()}")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # Get model from shareable. data_kind must be WEIGHTS.
+            if model_dxo.data and model_dxo.data_kind == DataKind.WEIGHTS:
+                model = model_dxo.data
+            else:
+                self.log_error(fl_ctx, f"Model Dex doesn't have data or is not of type DataKind.WEIGHTS. Unable  "
+                                       "to validate.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # The workflow provides MODEL_OWNER information in the shareable header.
+            model_name = shareable.get_header(AppConstants.MODEL_OWNER, "?")
+
+            # Print properties.
+            self.log_info(fl_ctx, f"Model: \n{model}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+            self.log_info(fl_ctx, f"Validating model from {model_name}.")
+
+            # Check abort signal regularly.
+            if abort_signal.triggered:
+                return self._abort_execution()
+
+            # Check if key exists in model
+            if NPConstants.NUMPY_KEY not in model:
+                self.log_error(fl_ctx, "numpy_key not in model. Unable to validate.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # Do some dummy validation.
+            random_epsilon = np.random.random()
+            self.log_info(
+                fl_ctx, f"Adding random epsilon {random_epsilon} in validation."
+            )
+            val_results = {}
+            np_data = model[NPConstants.NUMPY_KEY]
+            np_data = np.sum(np_data / np.max(np_data))
+            val_results["accuracy"] = np_data + random_epsilon
+
+            # Check abort signal regularly.
+            if abort_signal.triggered:
+                return self._abort_execution()
+
+            self.log_info(fl_ctx, f"Validation result: {val_results}")
+
+            # Create DXO for metrics and return shareable.
+            metric_dxo = DXO(data_kind=DataKind.METRICS, data=val_results)
+            return metric_dxo.to_shareable()
+
+        else:
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+            return shareable
+
+    def _abort_execution(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/test_apps/cross_val_one_client/config/config_fed_client.json b/test/app_testing/test_apps/cross_val_one_client/config/config_fed_client.json
new file mode 100755
index 0000000000..840d10aec0
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val_one_client/config/config_fed_client.json
@@ -0,0 +1,27 @@
+{
+  "format_version": 2,
+  "handlers": [],
+  "executors": [
+    {
+      "tasks": [
+        "train",
+        "submit_model"
+      ],
+      "executor": {
+        "path": "np_trainer.NPTrainer",
+        "args": {}
+      }
+    },
+    {
+      "tasks": [
+        "validate"
+      ],
+      "executor": {
+        "path": "np_validator.NPValidator"
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/cross_val_one_client/config/config_fed_server.json b/test/app_testing/test_apps/cross_val_one_client/config/config_fed_server.json
new file mode 100755
index 0000000000..e71550577d
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val_one_client/config/config_fed_server.json
@@ -0,0 +1,66 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "handlers": [],
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    },
+    {
+      "id": "model_locator",
+      "path": "np_model_locator.NPModelLocator",
+      "args": {}
+    },
+    {
+      "id": "formatter",
+      "path": "np_formatter.NPFormatter",
+      "args": {}
+    }
+  ],
+  "workflows": [
+    {
+      "id": "fed_avg_ctl",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 6000
+      }
+    },
+    {
+      "id": "cross_validation",
+      "path": "nvflare.app_common.workflows.cross_site_model_eval.CrossSiteModelEval",
+      "args": {
+        "model_locator_id": "model_locator",
+        "submit_model_timeout": 600,
+        "validation_timeout": 6000,
+        "cleanup_models": false,
+        "participating_clients": ["client_0"]
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/cross_val_one_client/custom/__init__.py b/test/app_testing/test_apps/cross_val_one_client/custom/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/app_testing/test_apps/cross_val_one_client/custom/constants.py b/test/app_testing/test_apps/cross_val_one_client/custom/constants.py
new file mode 100755
index 0000000000..7e0e5dc276
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val_one_client/custom/constants.py
@@ -0,0 +1,2 @@
+class NPConstants:
+    NUMPY_KEY = "numpy_key"
diff --git a/test/app_testing/test_apps/cross_val_one_client/custom/np_formatter.py b/test/app_testing/test_apps/cross_val_one_client/custom/np_formatter.py
new file mode 100755
index 0000000000..510618e231
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val_one_client/custom/np_formatter.py
@@ -0,0 +1,50 @@
+from nvflare.apis.dxo import from_bytes, DataKind
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.formatter import \
+    Formatter
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPFormatter(Formatter):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def format(self, fl_ctx: FLContext) -> str:
+        """The format function gets validation shareable locations from the dictionary. It loads each shareable, get the validation results and converts it into human readable string.
+
+        Args:
+            fl_ctx (FLContext): FLContext object.
+
+        Returns:
+            str: Human readable validation results.
+        """
+        # Get the val shareables
+        validation_shareables_dict = fl_ctx.get_prop(
+            AppConstants.VALIDATION_RESULT, {}
+        )
+
+        # Result dictionary
+        res = {}
+
+        try:
+            # This is a 2d dictionary with each evaluation result at validation_shareables_dict[data_client][model_client]
+            for data_client in validation_shareables_dict.keys():
+                validation_dict = validation_shareables_dict[data_client]
+                if validation_dict:
+                    res[data_client] = {}
+                    for model_name in validation_dict.keys():
+                        dxo_path = validation_dict[model_name]
+
+                        # Load the shareable
+                        metric_dxo = None
+                        with open(dxo_path, "rb") as f:
+                            metric_dxo = from_bytes(f.read())
+
+                        # Get metrics from shareable
+                        if metric_dxo and metric_dxo.data_kind== DataKind.METRICS:
+                            metrics = metric_dxo.data
+                            res[data_client][model_name] = metrics
+        except Exception as e:
+            self.log_error(fl_ctx, f"Exception: {e}")
+
+        return f"{res}"
diff --git a/test/app_testing/test_apps/cross_val_one_client/custom/np_model_locator.py b/test/app_testing/test_apps/cross_val_one_client/custom/np_model_locator.py
new file mode 100755
index 0000000000..de922c063e
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val_one_client/custom/np_model_locator.py
@@ -0,0 +1,56 @@
+import os
+from typing import List
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, DataKind
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model_locator import (
+    ModelLocator,
+)
+
+
+class NPModelLocator(ModelLocator):
+    SERVER_MODEL_NAME = "server"
+
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_file_name = model_name
+
+    def get_model_names(self, fl_ctx: FLContext) -> List[str]:
+        """Returns the list of model names that should be included from server in cross site validation.add()
+
+        Args:
+            fl_ctx (FLContext): FL Context object.
+
+        Returns:
+            List[str]: List of model names.
+        """
+        return [NPModelLocator.SERVER_MODEL_NAME]
+
+    def locate_model(self, model_name, fl_ctx: FLContext) -> DXO:
+        dxo = None
+        engine = fl_ctx.get_engine()
+
+        if model_name == NPModelLocator.SERVER_MODEL_NAME:
+            run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+            run_dir = engine.get_workspace().get_run_dir(run_number)
+            model_path = os.path.join(run_dir, self.model_dir)
+
+            model_load_path = os.path.join(model_path, self.model_file_name)
+            np_data = None
+            try:
+                np_data = np.load(model_load_path, allow_pickle=True)
+                self.log_info(fl_ctx, f"Loaded {model_name} model from {model_load_path}.")
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load NP Model: {e}.")
+
+            if np_data is not None:
+                weights = {NPConstants.NUMPY_KEY: np_data}
+                dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights, meta={})
+
+        return dxo
diff --git a/test/app_testing/test_apps/cross_val_one_client/custom/np_model_persistor.py b/test/app_testing/test_apps/cross_val_one_client/custom/np_model_persistor.py
new file mode 100755
index 0000000000..c8f16e7121
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val_one_client/custom/np_model_persistor.py
@@ -0,0 +1,62 @@
+import os
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable, make_model_learnable, ModelLearnableKey
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPModelPersistor(ModelPersistor):
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_name = model_name
+        self.default_data = np.array(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32
+        )
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        # Get start round
+        start_round = fl_ctx.get_prop(AppConstants.START_ROUND, 0)
+
+        # Create a new numpy model
+        if start_round > 0:
+            try:
+                engine = fl_ctx.get_engine()
+                run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+                run_dir = engine.get_workspace().get_run_dir(run_number)
+                model_path = os.path.join(run_dir, self.model_dir, self.model_name)
+                data = np.load(model_path)
+            except Exception as e:
+                self.log_info(
+                    fl_ctx,
+                    f"Unable to load model from {model_path}. Using default data instead."
+                )
+                data = self.default_data.copy()
+        else:
+            data = self.default_data.copy()
+
+        weights = {NPConstants.NUMPY_KEY: data}
+        model_learnable = make_model_learnable(weights, {})
+
+        self.logger.info(f"Loaded initial model: {model_learnable[ModelLearnableKey.WEIGHTS]}")
+        return model_learnable
+
+    def save_model(self, model: ModelLearnable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self.model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
diff --git a/test/app_testing/test_apps/cross_val_one_client/custom/np_trainer.py b/test/app_testing/test_apps/cross_val_one_client/custom/np_trainer.py
new file mode 100755
index 0000000000..d99911c44c
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val_one_client/custom/np_trainer.py
@@ -0,0 +1,196 @@
+import os
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, from_shareable, DataKind
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPTrainer(Executor):
+    def __init__(
+        self,
+        delta=1,
+        sleep_time=0,
+        train_task_name=AppConstants.TASK_TRAIN,
+        submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL,
+        model_name="best_numpy.npy",
+        model_dir="model",
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        if not isinstance(delta, int):
+            raise TypeError("")
+
+        self._delta = delta
+        self._model_name = model_name
+        self._model_dir = model_dir
+        self._sleep_time = sleep_time
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     # Create all major components here.
+        #     pass
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        #     pass
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any kind of tasks waiting should check abort_signal regularly
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._train_task_name:
+
+            try:
+                incoming_dxo = from_shareable(shareable)
+            except BaseException as e:
+                self.system_panic(f"Unable to convert shareable to model definition. Exception {e}", fl_ctx)
+                return self._get_exception_shareable()
+
+            current_round = shareable.get_header(AppConstants.CURRENT_ROUND, None)
+            total_rounds = shareable.get_header(AppConstants.NUM_ROUNDS, None)
+
+            # Get data from numpy shareable
+            if incoming_dxo.data_kind != DataKind.WEIGHTS:
+                self.system_panic("Model dex should be of kind DataKind.WEIGHTS.", fl_ctx)
+                return self._get_exception_shareable()
+
+            weights = incoming_dxo.data
+
+            self.log_info(fl_ctx, f"Incoming data kind: {incoming_dxo.data_kind}")
+            self.log_info(fl_ctx, f"Model: \n{weights}")
+            self.log_info(fl_ctx, f"Current Round: {current_round}")
+            self.log_info(fl_ctx, f"Total Rounds: {total_rounds}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Doing some dummy training.
+            if weights:
+                if NPConstants.NUMPY_KEY in weights:
+                    weights[NPConstants.NUMPY_KEY] += self._delta
+                else:
+                    self.log_error(fl_ctx, "numpy_key not found in model.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+                    return shareable
+            else:
+                self.log_error(fl_ctx, "No model weights found in shareable.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Save local model
+            try:
+                self._save_local_model(fl_ctx, weights)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Exception in saving local model: {e}.")
+
+            self.log_info(
+                fl_ctx,
+                f"Model after training: {weights}",
+            )
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            outgoing_dxo = DXO(data_kind=incoming_dxo.data_kind, data=weights, meta={})
+            return outgoing_dxo.to_shareable()
+        elif task_name == self._submit_model_task_name:
+            # Send the local model back to customer
+            weights = None
+            try:
+                weights = self._load_local_model(fl_ctx)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load model: {e}")
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            model_shareable = Shareable()
+            if weights:
+                incoming_dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights)
+                model_shareable = incoming_dxo.to_shareable()
+            else:
+                # Set return code.
+                model_shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+
+            return model_shareable
+        else:
+            # If unknown task name, set RC accordingly.
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+            return shareable
+
+    def _load_local_model(self, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+
+        model_load_path = os.path.join(model_path, self._model_name)
+        np_data = None
+        try:
+            np_data = np.load(model_load_path)
+        except Exception as e:
+            self.log_error(f"Unable to load local model: {np_data}")
+            return None
+
+        model = ModelLearnable()
+        model[NPConstants.NUMPY_KEY] = np_data
+
+        return model
+
+    def _save_local_model(self, fl_ctx: FLContext, model: dict):
+        # Save local model
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self._model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
+
+    def _get_exception_shareable(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/test_apps/cross_val_one_client/custom/np_validator.py b/test/app_testing/test_apps/cross_val_one_client/custom/np_validator.py
new file mode 100755
index 0000000000..a90dda3686
--- /dev/null
+++ b/test/app_testing/test_apps/cross_val_one_client/custom/np_validator.py
@@ -0,0 +1,115 @@
+import logging
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import from_shareable, DataKind, DXO
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPValidator(Executor):
+    def __init__(
+        self,
+        epsilon=1,
+        sleep_time=0,
+        validate_task_name=AppConstants.TASK_VALIDATION,
+    ):
+        super().__init__()
+
+        self.logger = logging.getLogger("NPValidator")
+        self._random_epsilon = epsilon
+        self._sleep_time = sleep_time
+        self._validate_task_name = validate_task_name
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any kind of tasks waiting should check abort_signal regularly
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._abort_execution()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._validate_task_name:
+
+            try:
+                model_dxo = from_shareable(shareable)
+            except Exception as e:
+                self.log_error(f"Unable to extract model dxo from shareable. Exception: {(e.__str__())}")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # Get model from shareable
+            if model_dxo.data and model_dxo.data_kind == DataKind.WEIGHTS:
+                model = model_dxo.data
+            else:
+                self.log_error(fl_ctx, f"Model Dex doesn't have data or is not of type DataKind.WEIGHTS. Unable  "
+                                       "to validate.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            model_name = shareable.get_header(AppConstants.MODEL_OWNER, "?")
+
+            self.log_info(fl_ctx, f"Model: \n{model}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+            self.log_info(fl_ctx, f"Validating model from {model_name}.")
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._abort_execution()
+
+            # Check if key exists in model
+            if NPConstants.NUMPY_KEY not in model:
+                self.log_error(fl_ctx, "numpy_key not in model. Unable to validate.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            random_epsilon = np.random.random()
+            self.log_info(
+                fl_ctx, f"Adding random epsilon {random_epsilon} in validation."
+            )
+
+            # Do some dummy validation here
+            val_results = {}
+            np_data = model[NPConstants.NUMPY_KEY]
+            np_data = np.sum(np_data / np.max(np_data))
+            val_results["accuracy"] = np_data + random_epsilon
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._abort_execution()
+
+            self.log_info(fl_ctx, f"Validation result: {val_results}")
+
+            metric_dxo = DXO(data_kind=DataKind.METRICS, data=val_results)
+
+            return metric_dxo.to_shareable()
+
+        else:
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+            return shareable
+
+    def _abort_execution(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/test_apps/cyclic/config/config_fed_client.json b/test/app_testing/test_apps/cyclic/config/config_fed_client.json
new file mode 100755
index 0000000000..7770537ff1
--- /dev/null
+++ b/test/app_testing/test_apps/cyclic/config/config_fed_client.json
@@ -0,0 +1,19 @@
+{
+  "format_version": 2,
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "trainer.SimpleTrainer",
+        "args": {
+          "epochs_per_round": 2
+        }
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "handlers": []
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/cyclic/config/config_fed_server.json b/test/app_testing/test_apps/cyclic/config/config_fed_server.json
new file mode 100755
index 0000000000..bbb3985e69
--- /dev/null
+++ b/test/app_testing/test_apps/cyclic/config/config_fed_server.json
@@ -0,0 +1,35 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "tf2_model_persistor.TF2ModelPersistor",
+      "args": {
+        "save_name": "tf2weights.pickle"
+      }
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    }
+  ],
+  "workflows": [
+    {
+      "id": "cyclic_ctl",
+      "path": "nvflare.app_common.workflows.cyclic_ctl.CyclicController",
+      "args": {
+        "num_rounds": 2,
+        "task_assignment_timeout": 8,
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "task_name": "train"
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/cyclic/config/environment.json b/test/app_testing/test_apps/cyclic/config/environment.json
new file mode 100755
index 0000000000..0967ef424b
--- /dev/null
+++ b/test/app_testing/test_apps/cyclic/config/environment.json
@@ -0,0 +1 @@
+{}
diff --git a/test/app_testing/test_apps/cyclic/custom/__init__.py b/test/app_testing/test_apps/cyclic/custom/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/test/app_testing/test_apps/cyclic/custom/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/test/app_testing/test_apps/cyclic/custom/tf2_model_persistor.py b/test/app_testing/test_apps/cyclic/custom/tf2_model_persistor.py
new file mode 100644
index 0000000000..721dc1c746
--- /dev/null
+++ b/test/app_testing/test_apps/cyclic/custom/tf2_model_persistor.py
@@ -0,0 +1,107 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pickle
+import json
+
+import tensorflow as tf
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from tf2_net import Net
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.abstract.model import make_model_learnable
+
+
+class TF2ModelPersistor(ModelPersistor):
+    def __init__(self, save_name="tf2_model.pkl"):
+        super().__init__()
+        self.save_name = save_name
+
+    def _initialize(self, fl_ctx: FLContext):
+        # get save path from FLContext
+        app_root = fl_ctx.get_prop(FLContextKey.APP_ROOT)
+        env_config = os.path.join(app_root, fl_ctx.get_prop(FLContextKey.ARGS).env)
+        with open(os.path.join(app_root, env_config)) as file:
+            env = json.load(file)
+
+        if env is not None:
+            if env.get("APP_CKPT_DIR", None):
+                fl_ctx.set_prop(
+                    AppConstants.LOG_DIR, env["APP_CKPT_DIR"], private=True, sticky=True
+                )
+            if env.get("APP_CKPT") is not None:
+                fl_ctx.set_prop(
+                    AppConstants.CKPT_PRELOAD_PATH,
+                    env["APP_CKPT"],
+                    private=True,
+                    sticky=True,
+                )
+
+        log_dir = fl_ctx.get_prop(AppConstants.LOG_DIR)
+        if log_dir:
+            self.log_dir = os.path.join(app_root, log_dir)
+        else:
+            self.log_dir = app_root
+        self._pkl_save_path = os.path.join(self.log_dir, self.save_name)
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        """
+            initialize and load the Model.
+
+        Args:
+            fl_ctx: FLContext
+
+        Returns:
+            Model object
+        """
+
+        if os.path.exists(self._pkl_save_path):
+            self.logger.info(f"Loading server weights")
+            with open(self._pkl_save_path, "rb") as f:
+                model_learnable = pickle.load(f)
+        else:
+            self.logger.info(f"Initializing server model")
+            network = Net()
+            loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+            network.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
+            _ = network(tf.keras.Input(shape=(28, 28)))
+            var_dict = {
+                network.get_layer(index=key).name: value
+                for key, value in enumerate(network.get_weights())
+            }
+            model_learnable = make_model_learnable(var_dict, dict())
+        return model_learnable
+
+    def handle_event(self, event: str, fl_ctx: FLContext):
+        if event == EventType.START_RUN:
+            self._initialize(fl_ctx)
+
+    def save_model(self, model_learnable: ModelLearnable, fl_ctx: FLContext):
+        """
+            persist the Model object
+
+        Args:
+            model: Model object
+            fl_ctx: FLContext
+        """
+        model_learnable_info = {k: str(type(v)) for k, v in model_learnable.items()}
+        self.logger.info(f"Saving aggregated server weights: \n {model_learnable_info}")
+        with open(self._pkl_save_path, "wb") as f:
+            pickle.dump(model_learnable, f)
diff --git a/test/app_testing/test_apps/cyclic/custom/tf2_net.py b/test/app_testing/test_apps/cyclic/custom/tf2_net.py
new file mode 100644
index 0000000000..526eeea0ca
--- /dev/null
+++ b/test/app_testing/test_apps/cyclic/custom/tf2_net.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tensorflow as tf
+
+
+class Net(tf.keras.Model):
+    def __init__(self):
+        super().__init__()
+        self.flatten = tf.keras.layers.Flatten(input_shape=(28, 28))
+        self.dense1 = tf.keras.layers.Dense(128, activation="relu")
+        self.dropout = tf.keras.layers.Dropout(0.2)
+        self.dense2 = tf.keras.layers.Dense(10)
+
+    def call(self, x):
+        x = self.flatten(x)
+        x = self.dense1(x)
+        x = self.dropout(x)
+        x = self.dense2(x)
+        return x
diff --git a/test/app_testing/test_apps/cyclic/custom/trainer.py b/test/app_testing/test_apps/cyclic/custom/trainer.py
new file mode 100644
index 0000000000..2355836207
--- /dev/null
+++ b/test/app_testing/test_apps/cyclic/custom/trainer.py
@@ -0,0 +1,142 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tensorflow as tf
+import numpy as np
+
+from nvflare.apis.dxo import DXO, DataKind, from_shareable
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.event_type import EventType
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.apis.signal import Signal
+
+from tf2_net import Net
+
+
+class SimpleTrainer(Executor):
+    def __init__(self, epochs_per_round):
+        super().__init__()
+        self.epochs_per_round = epochs_per_round
+        self.train_images, self.train_labels = None, None
+        self.test_images, self.test_labels = None, None
+        self.model = None
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.setup(fl_ctx)
+
+    def setup(self, fl_ctx: FLContext):
+        (self.train_images, self.train_labels), (
+            self.test_images,
+            self.test_labels,
+        ) = tf.keras.datasets.mnist.load_data()
+        self.train_images, self.test_images = (
+            self.train_images / 255.0,
+            self.test_images / 255.0,
+        )
+
+        # simulate separate datasets for each client by dividing MNIST dataset in half
+        client_name = fl_ctx.get_identity_name()
+        if client_name == "site-1":
+            self.train_images = self.train_images[: len(self.train_images) // 2]
+            self.train_labels = self.train_labels[: len(self.train_labels) // 2]
+            self.test_images = self.test_images[: len(self.test_images) // 2]
+            self.test_labels = self.test_labels[: len(self.test_labels) // 2]
+        elif client_name == "site-2":
+            self.train_images = self.train_images[len(self.train_images) // 2 :]
+            self.train_labels = self.train_labels[len(self.train_labels) // 2 :]
+            self.test_images = self.test_images[len(self.test_images) // 2 :]
+            self.test_labels = self.test_labels[len(self.test_labels) // 2 :]
+
+        model = Net()
+
+        loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+        model.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
+        _ = model(tf.keras.Input(shape=(28, 28)))
+        self.var_list = [
+            model.get_layer(index=index).name
+            for index in range(len(model.get_weights()))
+        ]
+        self.model = model
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        """
+        This function is an extended function from the super class.
+        As a supervised learning based trainer, the train function will run
+        evaluate and train engines based on model weights from `shareable`.
+        After finishing training, a new `Shareable` object will be submitted
+        to server for aggregation.
+
+        Args:
+            task_name: dispatched task
+            shareable: the `Shareable` object acheived from server.
+            fl_ctx: the `FLContext` object achieved from server.
+            abort_signal: if triggered, the training will be aborted.
+
+        Returns:
+            a new `Shareable` object to be submitted to server for aggregation.
+        """
+
+        # retrieve model weights download from server's shareable
+        if abort_signal.triggered:
+            return make_reply(ReturnCode.OK)
+
+        if task_name != "train":
+            return shareable
+
+        dxo = from_shareable(shareable)
+        model_weights = dxo.data
+
+        # use previous round's client weights to replace excluded layers from server
+        prev_weights = {
+            self.model.get_layer(index=key).name: value
+            for key, value in enumerate(self.model.get_weights())
+        }
+        print("dxo")
+        ordered_model_weights = {key: model_weights.get(key) for key in prev_weights}
+        for key in self.var_list:
+            value = ordered_model_weights.get(key)
+            if np.all(value == 0):
+                ordered_model_weights[key] = prev_weights[key]
+
+        # update local model weights with received weights
+        self.model.set_weights(list(ordered_model_weights.values()))
+
+        # adjust LR or other training time info as needed
+        # such as callback in the fit function
+        self.model.fit(
+            self.train_images,
+            self.train_labels,
+            epochs=self.epochs_per_round,
+            validation_data=(self.test_images, self.test_labels),
+        )
+
+        # report updated weights in shareable
+        weights = {
+            self.model.get_layer(index=key).name: value
+            for key, value in enumerate(self.model.get_weights())
+        }
+        dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights)
+
+        self.log_info(fl_ctx, "Local epochs finished. Returning shareable")
+
+        return dxo.update_shareable(shareable)
diff --git a/examples/hello-pt/resources/log.config b/test/app_testing/test_apps/cyclic/resources/log.config
similarity index 100%
rename from examples/hello-pt/resources/log.config
rename to test/app_testing/test_apps/cyclic/resources/log.config
diff --git a/test/app_testing/test_apps/filters/config/config_fed_client.json b/test/app_testing/test_apps/filters/config/config_fed_client.json
new file mode 100755
index 0000000000..09b5120622
--- /dev/null
+++ b/test/app_testing/test_apps/filters/config/config_fed_client.json
@@ -0,0 +1,54 @@
+{
+  "format_version": 2,
+  "handlers": [],
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "test.app_testing.np.np_trainer.NPTrainer",
+        "args": {}
+      }
+    }
+  ],
+  "task_result_filters": [
+    {
+      "tasks": ["train"],
+      "filters": [
+        {
+          "path": "nvflare.app_common.filters.exclude_vars.ExcludeVars",
+          "args": {}
+        },
+        {
+          "path": "nvflare.app_common.filters.percentile_privacy.PercentilePrivacy",
+          "args": {}
+        },
+        {
+          "path": "nvflare.app_common.filters.svt_privacy.SVTPrivacy",
+          "args": {}
+        }
+      ]
+    }
+  ],
+  "task_data_filters": [
+    {
+      "tasks": ["train"],
+      "filters": [
+        {
+          "path": "nvflare.app_common.filters.exclude_vars.ExcludeVars",
+          "args": {}
+        },
+        {
+          "path": "nvflare.app_common.filters.percentile_privacy.PercentilePrivacy",
+          "args": {}
+        },
+        {
+          "path": "nvflare.app_common.filters.svt_privacy.SVTPrivacy",
+          "args": {}
+        }
+      ]
+    }
+  ],
+  "components": []
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/filters/config/config_fed_server.json b/test/app_testing/test_apps/filters/config/config_fed_server.json
new file mode 100755
index 0000000000..fd99ad364a
--- /dev/null
+++ b/test/app_testing/test_apps/filters/config/config_fed_server.json
@@ -0,0 +1,81 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "handlers": [],
+  "task_data_filters": [
+    {
+      "tasks": ["train"],
+      "filters": [
+        {
+          "path": "nvflare.app_common.filters.exclude_vars.ExcludeVars",
+          "args": {}
+        },
+        {
+          "path": "nvflare.app_common.filters.percentile_privacy.PercentilePrivacy",
+          "args": {}
+        },
+        {
+          "path": "nvflare.app_common.filters.svt_privacy.SVTPrivacy",
+          "args": {}
+        }
+      ]
+    }
+  ],
+  "task_result_filters": [
+    {
+      "tasks": ["train"],
+      "filters": [
+        {
+          "path": "nvflare.app_common.filters.exclude_vars.ExcludeVars",
+          "args": {}
+        },
+        {
+          "path": "nvflare.app_common.filters.percentile_privacy.PercentilePrivacy",
+          "args": {}
+        },
+        {
+          "path": "nvflare.app_common.filters.svt_privacy.SVTPrivacy",
+          "args": {}
+        }
+      ]
+    }
+  ],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "test.app_testing.np.np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    }
+  ],
+  "workflows": [
+    {
+      "id": "scatter_and_gather",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 6000
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/global_model_eval/config/config_fed_client.json b/test/app_testing/test_apps/global_model_eval/config/config_fed_client.json
new file mode 100755
index 0000000000..840d10aec0
--- /dev/null
+++ b/test/app_testing/test_apps/global_model_eval/config/config_fed_client.json
@@ -0,0 +1,27 @@
+{
+  "format_version": 2,
+  "handlers": [],
+  "executors": [
+    {
+      "tasks": [
+        "train",
+        "submit_model"
+      ],
+      "executor": {
+        "path": "np_trainer.NPTrainer",
+        "args": {}
+      }
+    },
+    {
+      "tasks": [
+        "validate"
+      ],
+      "executor": {
+        "path": "np_validator.NPValidator"
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/global_model_eval/config/config_fed_server.json b/test/app_testing/test_apps/global_model_eval/config/config_fed_server.json
new file mode 100755
index 0000000000..8b45e828df
--- /dev/null
+++ b/test/app_testing/test_apps/global_model_eval/config/config_fed_server.json
@@ -0,0 +1,64 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "handlers": [],
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    },
+    {
+      "id": "model_locator",
+      "path": "np_model_locator.NPModelLocator",
+      "args": {}
+    },
+    {
+      "id": "formatter",
+      "path": "np_formatter.NPFormatter",
+      "args": {}
+    }
+  ],
+  "workflows": [
+    {
+      "id": "sag",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 6000
+      }
+    },
+    {
+      "id": "global_model_eval",
+      "path": "nvflare.app_common.workflows.global_model_eval.GlobalModelEval",
+      "args": {
+        "model_locator_id": "model_locator",
+        "validation_timeout": 6000,
+        "cleanup_models": false
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/global_model_eval/custom/__init__.py b/test/app_testing/test_apps/global_model_eval/custom/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/app_testing/test_apps/global_model_eval/custom/constants.py b/test/app_testing/test_apps/global_model_eval/custom/constants.py
new file mode 100755
index 0000000000..7e0e5dc276
--- /dev/null
+++ b/test/app_testing/test_apps/global_model_eval/custom/constants.py
@@ -0,0 +1,2 @@
+class NPConstants:
+    NUMPY_KEY = "numpy_key"
diff --git a/test/app_testing/test_apps/global_model_eval/custom/np_formatter.py b/test/app_testing/test_apps/global_model_eval/custom/np_formatter.py
new file mode 100755
index 0000000000..510618e231
--- /dev/null
+++ b/test/app_testing/test_apps/global_model_eval/custom/np_formatter.py
@@ -0,0 +1,50 @@
+from nvflare.apis.dxo import from_bytes, DataKind
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.formatter import \
+    Formatter
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPFormatter(Formatter):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def format(self, fl_ctx: FLContext) -> str:
+        """The format function gets validation shareable locations from the dictionary. It loads each shareable, get the validation results and converts it into human readable string.
+
+        Args:
+            fl_ctx (FLContext): FLContext object.
+
+        Returns:
+            str: Human readable validation results.
+        """
+        # Get the val shareables
+        validation_shareables_dict = fl_ctx.get_prop(
+            AppConstants.VALIDATION_RESULT, {}
+        )
+
+        # Result dictionary
+        res = {}
+
+        try:
+            # This is a 2d dictionary with each evaluation result at validation_shareables_dict[data_client][model_client]
+            for data_client in validation_shareables_dict.keys():
+                validation_dict = validation_shareables_dict[data_client]
+                if validation_dict:
+                    res[data_client] = {}
+                    for model_name in validation_dict.keys():
+                        dxo_path = validation_dict[model_name]
+
+                        # Load the shareable
+                        metric_dxo = None
+                        with open(dxo_path, "rb") as f:
+                            metric_dxo = from_bytes(f.read())
+
+                        # Get metrics from shareable
+                        if metric_dxo and metric_dxo.data_kind== DataKind.METRICS:
+                            metrics = metric_dxo.data
+                            res[data_client][model_name] = metrics
+        except Exception as e:
+            self.log_error(fl_ctx, f"Exception: {e}")
+
+        return f"{res}"
diff --git a/test/app_testing/test_apps/global_model_eval/custom/np_model_locator.py b/test/app_testing/test_apps/global_model_eval/custom/np_model_locator.py
new file mode 100755
index 0000000000..de922c063e
--- /dev/null
+++ b/test/app_testing/test_apps/global_model_eval/custom/np_model_locator.py
@@ -0,0 +1,56 @@
+import os
+from typing import List
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, DataKind
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model_locator import (
+    ModelLocator,
+)
+
+
+class NPModelLocator(ModelLocator):
+    SERVER_MODEL_NAME = "server"
+
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_file_name = model_name
+
+    def get_model_names(self, fl_ctx: FLContext) -> List[str]:
+        """Returns the list of model names that should be included from server in cross site validation.add()
+
+        Args:
+            fl_ctx (FLContext): FL Context object.
+
+        Returns:
+            List[str]: List of model names.
+        """
+        return [NPModelLocator.SERVER_MODEL_NAME]
+
+    def locate_model(self, model_name, fl_ctx: FLContext) -> DXO:
+        dxo = None
+        engine = fl_ctx.get_engine()
+
+        if model_name == NPModelLocator.SERVER_MODEL_NAME:
+            run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+            run_dir = engine.get_workspace().get_run_dir(run_number)
+            model_path = os.path.join(run_dir, self.model_dir)
+
+            model_load_path = os.path.join(model_path, self.model_file_name)
+            np_data = None
+            try:
+                np_data = np.load(model_load_path, allow_pickle=True)
+                self.log_info(fl_ctx, f"Loaded {model_name} model from {model_load_path}.")
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load NP Model: {e}.")
+
+            if np_data is not None:
+                weights = {NPConstants.NUMPY_KEY: np_data}
+                dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights, meta={})
+
+        return dxo
diff --git a/test/app_testing/test_apps/global_model_eval/custom/np_model_persistor.py b/test/app_testing/test_apps/global_model_eval/custom/np_model_persistor.py
new file mode 100755
index 0000000000..c8f16e7121
--- /dev/null
+++ b/test/app_testing/test_apps/global_model_eval/custom/np_model_persistor.py
@@ -0,0 +1,62 @@
+import os
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable, make_model_learnable, ModelLearnableKey
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPModelPersistor(ModelPersistor):
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_name = model_name
+        self.default_data = np.array(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32
+        )
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        # Get start round
+        start_round = fl_ctx.get_prop(AppConstants.START_ROUND, 0)
+
+        # Create a new numpy model
+        if start_round > 0:
+            try:
+                engine = fl_ctx.get_engine()
+                run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+                run_dir = engine.get_workspace().get_run_dir(run_number)
+                model_path = os.path.join(run_dir, self.model_dir, self.model_name)
+                data = np.load(model_path)
+            except Exception as e:
+                self.log_info(
+                    fl_ctx,
+                    f"Unable to load model from {model_path}. Using default data instead."
+                )
+                data = self.default_data.copy()
+        else:
+            data = self.default_data.copy()
+
+        weights = {NPConstants.NUMPY_KEY: data}
+        model_learnable = make_model_learnable(weights, {})
+
+        self.logger.info(f"Loaded initial model: {model_learnable[ModelLearnableKey.WEIGHTS]}")
+        return model_learnable
+
+    def save_model(self, model: ModelLearnable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self.model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
diff --git a/test/app_testing/test_apps/global_model_eval/custom/np_trainer.py b/test/app_testing/test_apps/global_model_eval/custom/np_trainer.py
new file mode 100755
index 0000000000..d99911c44c
--- /dev/null
+++ b/test/app_testing/test_apps/global_model_eval/custom/np_trainer.py
@@ -0,0 +1,196 @@
+import os
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, from_shareable, DataKind
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPTrainer(Executor):
+    def __init__(
+        self,
+        delta=1,
+        sleep_time=0,
+        train_task_name=AppConstants.TASK_TRAIN,
+        submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL,
+        model_name="best_numpy.npy",
+        model_dir="model",
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        if not isinstance(delta, int):
+            raise TypeError("")
+
+        self._delta = delta
+        self._model_name = model_name
+        self._model_dir = model_dir
+        self._sleep_time = sleep_time
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     # Create all major components here.
+        #     pass
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        #     pass
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any kind of tasks waiting should check abort_signal regularly
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._train_task_name:
+
+            try:
+                incoming_dxo = from_shareable(shareable)
+            except BaseException as e:
+                self.system_panic(f"Unable to convert shareable to model definition. Exception {e}", fl_ctx)
+                return self._get_exception_shareable()
+
+            current_round = shareable.get_header(AppConstants.CURRENT_ROUND, None)
+            total_rounds = shareable.get_header(AppConstants.NUM_ROUNDS, None)
+
+            # Get data from numpy shareable
+            if incoming_dxo.data_kind != DataKind.WEIGHTS:
+                self.system_panic("Model dex should be of kind DataKind.WEIGHTS.", fl_ctx)
+                return self._get_exception_shareable()
+
+            weights = incoming_dxo.data
+
+            self.log_info(fl_ctx, f"Incoming data kind: {incoming_dxo.data_kind}")
+            self.log_info(fl_ctx, f"Model: \n{weights}")
+            self.log_info(fl_ctx, f"Current Round: {current_round}")
+            self.log_info(fl_ctx, f"Total Rounds: {total_rounds}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Doing some dummy training.
+            if weights:
+                if NPConstants.NUMPY_KEY in weights:
+                    weights[NPConstants.NUMPY_KEY] += self._delta
+                else:
+                    self.log_error(fl_ctx, "numpy_key not found in model.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+                    return shareable
+            else:
+                self.log_error(fl_ctx, "No model weights found in shareable.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Save local model
+            try:
+                self._save_local_model(fl_ctx, weights)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Exception in saving local model: {e}.")
+
+            self.log_info(
+                fl_ctx,
+                f"Model after training: {weights}",
+            )
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            outgoing_dxo = DXO(data_kind=incoming_dxo.data_kind, data=weights, meta={})
+            return outgoing_dxo.to_shareable()
+        elif task_name == self._submit_model_task_name:
+            # Send the local model back to customer
+            weights = None
+            try:
+                weights = self._load_local_model(fl_ctx)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load model: {e}")
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            model_shareable = Shareable()
+            if weights:
+                incoming_dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights)
+                model_shareable = incoming_dxo.to_shareable()
+            else:
+                # Set return code.
+                model_shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+
+            return model_shareable
+        else:
+            # If unknown task name, set RC accordingly.
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+            return shareable
+
+    def _load_local_model(self, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+
+        model_load_path = os.path.join(model_path, self._model_name)
+        np_data = None
+        try:
+            np_data = np.load(model_load_path)
+        except Exception as e:
+            self.log_error(f"Unable to load local model: {np_data}")
+            return None
+
+        model = ModelLearnable()
+        model[NPConstants.NUMPY_KEY] = np_data
+
+        return model
+
+    def _save_local_model(self, fl_ctx: FLContext, model: dict):
+        # Save local model
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self._model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
+
+    def _get_exception_shareable(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/test_apps/global_model_eval/custom/np_validator.py b/test/app_testing/test_apps/global_model_eval/custom/np_validator.py
new file mode 100755
index 0000000000..a90dda3686
--- /dev/null
+++ b/test/app_testing/test_apps/global_model_eval/custom/np_validator.py
@@ -0,0 +1,115 @@
+import logging
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import from_shareable, DataKind, DXO
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPValidator(Executor):
+    def __init__(
+        self,
+        epsilon=1,
+        sleep_time=0,
+        validate_task_name=AppConstants.TASK_VALIDATION,
+    ):
+        super().__init__()
+
+        self.logger = logging.getLogger("NPValidator")
+        self._random_epsilon = epsilon
+        self._sleep_time = sleep_time
+        self._validate_task_name = validate_task_name
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any kind of tasks waiting should check abort_signal regularly
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._abort_execution()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._validate_task_name:
+
+            try:
+                model_dxo = from_shareable(shareable)
+            except Exception as e:
+                self.log_error(f"Unable to extract model dxo from shareable. Exception: {(e.__str__())}")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            # Get model from shareable
+            if model_dxo.data and model_dxo.data_kind == DataKind.WEIGHTS:
+                model = model_dxo.data
+            else:
+                self.log_error(fl_ctx, f"Model Dex doesn't have data or is not of type DataKind.WEIGHTS. Unable  "
+                                       "to validate.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            model_name = shareable.get_header(AppConstants.MODEL_OWNER, "?")
+
+            self.log_info(fl_ctx, f"Model: \n{model}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+            self.log_info(fl_ctx, f"Validating model from {model_name}.")
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._abort_execution()
+
+            # Check if key exists in model
+            if NPConstants.NUMPY_KEY not in model:
+                self.log_error(fl_ctx, "numpy_key not in model. Unable to validate.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            random_epsilon = np.random.random()
+            self.log_info(
+                fl_ctx, f"Adding random epsilon {random_epsilon} in validation."
+            )
+
+            # Do some dummy validation here
+            val_results = {}
+            np_data = model[NPConstants.NUMPY_KEY]
+            np_data = np.sum(np_data / np.max(np_data))
+            val_results["accuracy"] = np_data + random_epsilon
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._abort_execution()
+
+            self.log_info(fl_ctx, f"Validation result: {val_results}")
+
+            metric_dxo = DXO(data_kind=DataKind.METRICS, data=val_results)
+
+            return metric_dxo.to_shareable()
+
+        else:
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+            return shareable
+
+    def _abort_execution(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/test_apps/pt/config/config_fed_client.json b/test/app_testing/test_apps/pt/config/config_fed_client.json
new file mode 100644
index 0000000000..26847da620
--- /dev/null
+++ b/test/app_testing/test_apps/pt/config/config_fed_client.json
@@ -0,0 +1,32 @@
+{
+  "format_version": 2,
+
+  "handlers": [
+  ],
+  "executors": [
+    {
+      "tasks": ["train", "submit_model"],
+      "executor": {
+        "path": "cifar10trainer.Cifar10Trainer",
+        "args": {
+          "lr": 0.01,
+          "epochs": 1
+        }
+      }
+    },
+    {
+      "tasks": ["validate"],
+      "executor": {
+        "path": "cifar10validator.Cifar10Validator",
+        "args": {
+        }
+      }
+    }
+  ],
+  "task_result_filters": [
+  ],
+  "task_data_filters": [
+  ],
+  "components": [
+  ]
+}
diff --git a/test/app_testing/test_apps/pt/config/config_fed_server.json b/test/app_testing/test_apps/pt/config/config_fed_server.json
new file mode 100644
index 0000000000..42c8c98c55
--- /dev/null
+++ b/test/app_testing/test_apps/pt/config/config_fed_server.json
@@ -0,0 +1,70 @@
+{
+  "format_version": 2,
+
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "handlers": [],
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "name": "PTFileModelPersistor",
+      "args": {
+        "model": {
+          "path": "net.SimpleNetwork"
+        }
+      }
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    },
+    {
+      "id": "model_locator",
+      "path": "pt_model_locator.PTModelLocator",
+      "args": {
+
+      }
+    },
+    {
+      "id": "json_generator",
+      "path": "validation_json_generator.ValidationJsonGenerator",
+      "args": {
+      }
+    }
+  ],
+  "workflows": [
+      {
+        "id": "scatter_and_gather",
+        "name": "ScatterAndGather",
+        "args": {
+            "min_clients" : 2,
+            "num_rounds" : 2,
+            "start_round": 0,
+            "wait_time_after_min_received": 10,
+            "aggregator_id": "aggregator",
+            "persistor_id": "persistor",
+            "shareable_generator_id": "shareable_generator",
+            "train_task_name": "train",
+            "train_timeout": 0
+        }
+      },
+      {
+        "id": "cross_site_validate",
+        "name": "CrossSiteModelEval",
+        "args": {
+          "model_locator_id": "model_locator"
+        }
+      }
+  ]
+}
diff --git a/test/app_testing/test_apps/pt/config/environment.json b/test/app_testing/test_apps/pt/config/environment.json
new file mode 100644
index 0000000000..c8a1433f90
--- /dev/null
+++ b/test/app_testing/test_apps/pt/config/environment.json
@@ -0,0 +1,3 @@
+{
+  "APP_CKPT_DIR": "models"
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/pt/custom/cifar10trainer.py b/test/app_testing/test_apps/pt/custom/cifar10trainer.py
new file mode 100644
index 0000000000..83833d5fad
--- /dev/null
+++ b/test/app_testing/test_apps/pt/custom/cifar10trainer.py
@@ -0,0 +1,149 @@
+import os.path
+
+import torch
+import torchvision
+from torch import nn
+from torchvision import transforms
+
+from nvflare.apis.dxo import from_shareable, DXO, DataKind, MetaKey
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode, ReservedKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.model import make_model_learnable, model_learnable_to_dxo
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.pt.pt_fed_utils import PTModelPersistenceFormatManager
+from test.app_testing.test_apps.pt.custom.constants import PTConstants
+from test.app_testing.test_apps.pt.custom.net import SimpleNetwork
+
+
+class Cifar10Trainer(Executor):
+
+    def __init__(self, lr=0.01, epochs=5, train_task_name=AppConstants.TASK_TRAIN,
+                 submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL, exclude_vars=None):
+        super(Cifar10Trainer, self).__init__()
+
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+        # Training setup
+        self.model = SimpleNetwork()
+        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+        self.model.to(self.device)
+        self.loss = nn.CrossEntropyLoss()
+        self.optimizer = torch.optim.SGD(self.model.parameters(), lr=lr, momentum=0.9)
+        self.lr = lr
+        self.transforms = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
+        ])
+        self.epochs = epochs
+        self.exclude_vars = exclude_vars
+
+        # Cifar10 dataset for training.
+        self.train_dataset = torchvision.datasets.CIFAR10(root='~/data', transform=self.transforms,
+                                                          download=True, train=True)
+        self.train_loader = torch.utils.data.DataLoader(
+            self.train_dataset, batch_size=4, shuffle=True
+        )
+        self.n_iterations = len(self.train_loader)
+
+        # # Setup the persistence manager to save PT model.
+        self.default_train_conf = {"train": {"model": type(self.model).__name__}}
+        self.persistence_manager = PTModelPersistenceFormatManager(data=self.model.state_dict(),
+                                                                   default_train_conf=self.default_train_conf)
+
+    def local_train(self, fl_ctx, weights):
+        # Set the model weights
+        self.model.load_state_dict(state_dict=weights)
+
+        # Basic training
+        self.model.train()
+        for epoch in range(self.epochs):
+            running_loss = 0
+            for i, batch in enumerate(self.train_loader):
+                images, labels = batch[0].to(self.device), batch[1].to(self.device)
+                self.optimizer.zero_grad()
+
+                predictions = self.model(images)
+                cost = self.loss(predictions, labels)
+                cost.backward()
+                self.optimizer.step()
+
+                running_loss += (cost.cpu().detach().numpy()/images.size()[0])
+                if i % 3000 == 0:
+                    self.log_info(fl_ctx, f"Epoch: {epoch}/{self.epochs}, Iteration: {i}, "
+                                          f"Loss: {running_loss/3000}")
+                    running_loss = 0.0
+
+    def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable:
+        try:
+            if task_name == self._train_task_name:
+                # Get model weights
+                dxo = from_shareable(shareable)
+
+                # Check if dxo is valid.
+                if not isinstance(dxo, DXO):
+                    self.log_exception(fl_ctx, f"dxo excepted type DXO. Got {type(dxo)} instead.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                    return shareable
+
+                # Ensure data kind is weights.
+                if not dxo.data_kind == DataKind.WEIGHTS:
+                    self.log_exception(fl_ctx, f"data_kind expected WEIGHTS but got {dxo.data_kind} instead.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                    return shareable
+
+                # Convert weights to tensor. Run training
+                torch_weights = {k: torch.as_tensor(v) for k, v in dxo.data.items()}
+                self.local_train(fl_ctx, torch_weights)
+
+                self.save_local_model(fl_ctx)
+
+                # Get the new state dict and send as weights
+                new_weights = self.model.state_dict()
+                new_weights = {k: v.cpu().numpy() for k, v in new_weights.items()}
+
+                outgoing_dxo = DXO(data_kind=DataKind.WEIGHTS, data=new_weights,
+                                   meta={MetaKey.NUM_STEPS_CURRENT_ROUND: self.n_iterations})
+                return outgoing_dxo.to_shareable()
+            elif task_name == self._submit_model_task_name:
+                # Load local model
+                ml = self.load_local_model(fl_ctx)
+
+                # Get the model parameters and create dxo from it
+                dxo = model_learnable_to_dxo(ml)
+                return dxo.to_shareable()
+            else:
+                shareable = Shareable()
+                shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+                return shareable
+        except:
+            self.log_exception(fl_ctx, f"Exception in simple trainer.")
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+            return shareable
+
+    def save_local_model(self, fl_ctx: FLContext):
+        run_dir = fl_ctx.get_engine().get_workspace().get_run_dir(fl_ctx.get_prop(ReservedKey.RUN_NUM))
+        models_dir = os.path.join(run_dir, 'models')
+        if not os.path.exists(models_dir):
+            os.makedirs(models_dir)
+        model_path = os.path.join(models_dir, PTConstants.PTLocalModelName)
+
+        ml = make_model_learnable(self.model.state_dict(), {})
+        self.persistence_manager.update(ml)
+        torch.save(self.persistence_manager.to_persistence_dict(), model_path)
+
+    def load_local_model(self, fl_ctx: FLContext):
+        run_dir = fl_ctx.get_engine().get_workspace().get_run_dir(fl_ctx.get_prop(ReservedKey.RUN_NUM))
+        models_dir = os.path.join(run_dir, 'models')
+        if not os.path.exists(models_dir):
+            return None
+        model_path = os.path.join(models_dir, PTConstants.PTLocalModelName)
+
+        self.persistence_manager = PTModelPersistenceFormatManager(data=torch.load(model_path),
+                                                                   default_train_conf=self.default_train_conf)
+        ml = self.persistence_manager.to_model_learnable(exclude_vars=self.exclude_vars)
+        return ml
diff --git a/test/app_testing/test_apps/pt/custom/cifar10validator.py b/test/app_testing/test_apps/pt/custom/cifar10validator.py
new file mode 100644
index 0000000000..fccc8e59f3
--- /dev/null
+++ b/test/app_testing/test_apps/pt/custom/cifar10validator.py
@@ -0,0 +1,92 @@
+import torch
+import torchvision.datasets
+from torchvision import transforms
+
+from net import SimpleNetwork
+from nvflare.apis.dxo import from_shareable, DataKind, DXO
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.app_constant import AppConstants
+
+
+class Cifar10Validator(Executor):
+    
+    def __init__(self, validate_task_name=AppConstants.TASK_VALIDATION):
+        super(Cifar10Validator, self).__init__()
+
+        self._validate_task_name = validate_task_name
+
+        # Setup the model
+        self.model = SimpleNetwork()
+        self.transforms = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
+        ])
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.model.to(self.device)
+
+        # Preparing the dataset for testing.
+        self.test_data = torchvision.datasets.CIFAR10(root='~/data', train=False, transform=self.transforms)
+        self.test_loader = torch.utils.data.DataLoader(self.test_data, batch_size=4, shuffle=False)
+
+    def execute(self, task_name: str, shareable: Shareable, fl_ctx: FLContext, abort_signal: Signal) -> Shareable:
+        out_shareable = Shareable()
+        if task_name == self._validate_task_name:
+            try:
+                dxo = from_shareable(shareable)
+
+                # Check if dxo is valid.
+                if not dxo:
+                    self.log_exception(fl_ctx, "DXO invalid")
+                    out_shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                    return out_shareable
+
+                # Ensure data_kind is weights.
+                if not dxo.data_kind == DataKind.WEIGHTS:
+                    self.log_exception(fl_ctx, f"DXO is of type {dxo.data_kind} but expected type WEIGHTS.")
+                    out_shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                    return out_shareable
+
+                # Extract weights and ensure they are tensor.
+                model_owner = shareable.get_header(AppConstants.MODEL_OWNER, "?")
+                weights = dxo.data
+                weights = {k: torch.as_tensor(v, device=self.device) for k, v in weights.items()}
+
+                # Get validation accuracy
+                val_accuracy = self.do_validation(weights)
+                self.log_info(fl_ctx, f"Accuracy when validating {model_owner}'s model on"
+                                      f" {fl_ctx.get_identity_name()}"f's data: {val_accuracy}')
+
+                dxo = DXO(data_kind=DataKind.METRICS, data={'val_acc': val_accuracy})
+                return dxo.to_shareable()
+            except:
+                self.log_exception(fl_ctx, f"Exception in validating model from {model_owner}")
+                out_shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return out_shareable
+        else:
+            out_shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+            return out_shareable
+
+    def do_validation(self, weights):
+        self.model.load_state_dict(weights)
+
+        self.model.eval()
+
+        correct = 0
+        total = 0
+        with torch.no_grad():
+            for i, (images, labels) in enumerate(self.test_loader):
+                images, labels = images.to(self.device), labels.to(self.device)
+                output = self.model(images)
+
+                _, pred_label = torch.max(output, 1)
+
+                correct += (pred_label == labels).sum().item()
+                total += images.size()[0]
+
+            metric = correct/float(total)
+
+        return metric
diff --git a/test/app_testing/test_apps/pt/custom/constants.py b/test/app_testing/test_apps/pt/custom/constants.py
new file mode 100644
index 0000000000..cdb9cf1008
--- /dev/null
+++ b/test/app_testing/test_apps/pt/custom/constants.py
@@ -0,0 +1,6 @@
+
+
+class PTConstants:
+    PTServerName = "server"
+    PTFileModelName = "FL_global_model.pt"
+    PTLocalModelName = "local_model.pt"
diff --git a/test/app_testing/test_apps/pt/custom/net.py b/test/app_testing/test_apps/pt/custom/net.py
new file mode 100644
index 0000000000..fd1dfe6d54
--- /dev/null
+++ b/test/app_testing/test_apps/pt/custom/net.py
@@ -0,0 +1,24 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class SimpleNetwork(nn.Module):
+    def __init__(self):
+        super(SimpleNetwork, self).__init__()
+
+        self.conv1 = nn.Conv2d(3, 6, 5)
+        self.pool = nn.MaxPool2d(2, 2)
+        self.conv2 = nn.Conv2d(6, 16, 5)
+        self.fc1 = nn.Linear(16 * 5 * 5, 120)
+        self.fc2 = nn.Linear(120, 84)
+        self.fc3 = nn.Linear(84, 10)
+
+    def forward(self, x):
+        x = self.pool(F.relu(self.conv1(x)))
+        x = self.pool(F.relu(self.conv2(x)))
+        x = torch.flatten(x, 1)  # flatten all dimensions except batch
+        x = F.relu(self.fc1(x))
+        x = F.relu(self.fc2(x))
+        x = self.fc3(x)
+        return x
diff --git a/test/app_testing/test_apps/pt/custom/pt_model_locator.py b/test/app_testing/test_apps/pt/custom/pt_model_locator.py
new file mode 100644
index 0000000000..dead0d0599
--- /dev/null
+++ b/test/app_testing/test_apps/pt/custom/pt_model_locator.py
@@ -0,0 +1,51 @@
+import os
+from typing import List
+
+import torch.cuda
+
+from app_testing.test_apps.pt.custom.net import SimpleNetwork
+from nvflare.apis.dxo import DXO
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import model_learnable_to_dxo
+from nvflare.app_common.abstract.model_locator import ModelLocator
+from nvflare.app_common.pt.pt_fed_utils import PTModelPersistenceFormatManager
+from test.app_testing.test_apps.pt.custom.constants import PTConstants
+
+
+class PTModelLocator(ModelLocator):
+
+    def __init__(self, exclude_vars=None, model=None):
+        super(PTModelLocator, self).__init__()
+
+        self.model = SimpleNetwork()
+        self.exclude_vars = exclude_vars
+
+    def get_model_names(self, fl_ctx: FLContext) -> List[str]:
+        return [PTConstants.PTServerName]
+
+    def locate_model(self, model_name, fl_ctx: FLContext) -> DXO:
+        if model_name == PTConstants.PTServerName:
+            server_run_dir = fl_ctx.get_engine().get_workspace().get_app_dir(fl_ctx.get_run_number())
+            model_path = os.path.join(server_run_dir, "models", PTConstants.PTFileModelName)
+            if not os.path.exists(model_path):
+                return None
+
+            # Load the torch model
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+            data = torch.load(model_path, map_location=device)
+
+            # Setup the persistence manager.
+            if self.model:
+                default_train_conf = {"train": {"model": type(self.model).__name__}}
+            else:
+                default_train_conf = None
+
+            # Use persistence manager to get learnable
+            persistence_manager = PTModelPersistenceFormatManager(data, default_train_conf=default_train_conf)
+            ml = persistence_manager.to_model_learnable(exclude_vars=None)
+
+            # Create dxo and return
+            return model_learnable_to_dxo(ml)
+        else:
+            self.log_exception(fl_ctx, f"PTModelLocator doesn't recognize name: {model_name}")
+            return None
diff --git a/test/app_testing/test_apps/pt/custom/validation_json_generator.py b/test/app_testing/test_apps/pt/custom/validation_json_generator.py
new file mode 100644
index 0000000000..93b62b3904
--- /dev/null
+++ b/test/app_testing/test_apps/pt/custom/validation_json_generator.py
@@ -0,0 +1,56 @@
+import json
+import os.path
+
+from nvflare.apis.dxo import from_shareable, DataKind
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.app_constant import AppConstants
+
+
+class ValidationJsonGenerator(FLComponent):
+
+    def __init__(self, results_dir="cross_val_results", json_file_name="cross_val.json"):
+        super(ValidationJsonGenerator, self).__init__()
+
+        self.results_dir = results_dir
+        self.val_results = {}
+        self.json_file_name = json_file_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.val_results.clear()
+        elif event_type == AppConstants.VALIDATION_RESULT_RECEIVED:
+            model_owner = fl_ctx.get_prop(AppConstants.MODEL_OWNER, None)
+            data_client = fl_ctx.get_prop(AppConstants.DATA_CLIENT, None)
+            val_results = fl_ctx.get_prop(AppConstants.VALIDATION_RESULT, None)
+
+            if not model_owner:
+                self.log_error(fl_ctx, "model_owner unknown. Validation result will not be saved to json")
+            if not data_client:
+                self.log_error(fl_ctx, "data_client unknown. Validation result will not be saved to json")
+
+            if val_results:
+                try:
+                    dxo = from_shareable(val_results)
+                    dxo.validate()
+
+                    if dxo.data_kind == DataKind.METRICS:
+                        if data_client not in self.val_results:
+                            self.val_results[data_client] = {}
+                        self.val_results[data_client][model_owner] = dxo.data
+                    else:
+                        self.log_error(fl_ctx, f"Expected dxo of kind METRICS but got {dxo.data_kind} instead.")
+                except:
+                    self.log_exception(fl_ctx, f"Exception in handling validation result.")
+            else:
+                self.log_error(fl_ctx, "Validation result not found.")
+        elif event_type == EventType.END_RUN:
+            run_dir = fl_ctx.get_engine().get_workspace().get_run_dir(fl_ctx.get_run_number())
+            cross_val_res_dir = os.path.join(run_dir, self.results_dir)
+            if not os.path.exists(cross_val_res_dir):
+                os.makedirs(cross_val_res_dir)
+
+            res_file_path = os.path.join(cross_val_res_dir, self.json_file_name)
+            with open(res_file_path, 'w') as f:
+                json.dump(self.val_results, f)
diff --git a/test/app_testing/test_apps/sag/config/config_fed_client.json b/test/app_testing/test_apps/sag/config/config_fed_client.json
new file mode 100755
index 0000000000..6a42778034
--- /dev/null
+++ b/test/app_testing/test_apps/sag/config/config_fed_client.json
@@ -0,0 +1,18 @@
+{
+  "format_version": 2,
+  "handlers": [],
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "np_trainer.NPTrainer",
+        "args": {}
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/sag/config/config_fed_server.json b/test/app_testing/test_apps/sag/config/config_fed_server.json
new file mode 100755
index 0000000000..79b48d85c6
--- /dev/null
+++ b/test/app_testing/test_apps/sag/config/config_fed_server.json
@@ -0,0 +1,45 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "handlers": [],
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    }
+  ],
+  "workflows": [
+    {
+      "id": "scatter_and_gather",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 6000
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/sag/custom/__init__.py b/test/app_testing/test_apps/sag/custom/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/app_testing/test_apps/sag/custom/constants.py b/test/app_testing/test_apps/sag/custom/constants.py
new file mode 100755
index 0000000000..7e0e5dc276
--- /dev/null
+++ b/test/app_testing/test_apps/sag/custom/constants.py
@@ -0,0 +1,2 @@
+class NPConstants:
+    NUMPY_KEY = "numpy_key"
diff --git a/test/app_testing/test_apps/sag/custom/np_model_persistor.py b/test/app_testing/test_apps/sag/custom/np_model_persistor.py
new file mode 100755
index 0000000000..c8f16e7121
--- /dev/null
+++ b/test/app_testing/test_apps/sag/custom/np_model_persistor.py
@@ -0,0 +1,62 @@
+import os
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable, make_model_learnable, ModelLearnableKey
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPModelPersistor(ModelPersistor):
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_name = model_name
+        self.default_data = np.array(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32
+        )
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        # Get start round
+        start_round = fl_ctx.get_prop(AppConstants.START_ROUND, 0)
+
+        # Create a new numpy model
+        if start_round > 0:
+            try:
+                engine = fl_ctx.get_engine()
+                run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+                run_dir = engine.get_workspace().get_run_dir(run_number)
+                model_path = os.path.join(run_dir, self.model_dir, self.model_name)
+                data = np.load(model_path)
+            except Exception as e:
+                self.log_info(
+                    fl_ctx,
+                    f"Unable to load model from {model_path}. Using default data instead."
+                )
+                data = self.default_data.copy()
+        else:
+            data = self.default_data.copy()
+
+        weights = {NPConstants.NUMPY_KEY: data}
+        model_learnable = make_model_learnable(weights, {})
+
+        self.logger.info(f"Loaded initial model: {model_learnable[ModelLearnableKey.WEIGHTS]}")
+        return model_learnable
+
+    def save_model(self, model: ModelLearnable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self.model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
diff --git a/test/app_testing/test_apps/sag/custom/np_trainer.py b/test/app_testing/test_apps/sag/custom/np_trainer.py
new file mode 100755
index 0000000000..d99911c44c
--- /dev/null
+++ b/test/app_testing/test_apps/sag/custom/np_trainer.py
@@ -0,0 +1,196 @@
+import os
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, from_shareable, DataKind
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPTrainer(Executor):
+    def __init__(
+        self,
+        delta=1,
+        sleep_time=0,
+        train_task_name=AppConstants.TASK_TRAIN,
+        submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL,
+        model_name="best_numpy.npy",
+        model_dir="model",
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        if not isinstance(delta, int):
+            raise TypeError("")
+
+        self._delta = delta
+        self._model_name = model_name
+        self._model_dir = model_dir
+        self._sleep_time = sleep_time
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     # Create all major components here.
+        #     pass
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        #     pass
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any kind of tasks waiting should check abort_signal regularly
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._train_task_name:
+
+            try:
+                incoming_dxo = from_shareable(shareable)
+            except BaseException as e:
+                self.system_panic(f"Unable to convert shareable to model definition. Exception {e}", fl_ctx)
+                return self._get_exception_shareable()
+
+            current_round = shareable.get_header(AppConstants.CURRENT_ROUND, None)
+            total_rounds = shareable.get_header(AppConstants.NUM_ROUNDS, None)
+
+            # Get data from numpy shareable
+            if incoming_dxo.data_kind != DataKind.WEIGHTS:
+                self.system_panic("Model dex should be of kind DataKind.WEIGHTS.", fl_ctx)
+                return self._get_exception_shareable()
+
+            weights = incoming_dxo.data
+
+            self.log_info(fl_ctx, f"Incoming data kind: {incoming_dxo.data_kind}")
+            self.log_info(fl_ctx, f"Model: \n{weights}")
+            self.log_info(fl_ctx, f"Current Round: {current_round}")
+            self.log_info(fl_ctx, f"Total Rounds: {total_rounds}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Doing some dummy training.
+            if weights:
+                if NPConstants.NUMPY_KEY in weights:
+                    weights[NPConstants.NUMPY_KEY] += self._delta
+                else:
+                    self.log_error(fl_ctx, "numpy_key not found in model.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+                    return shareable
+            else:
+                self.log_error(fl_ctx, "No model weights found in shareable.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Save local model
+            try:
+                self._save_local_model(fl_ctx, weights)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Exception in saving local model: {e}.")
+
+            self.log_info(
+                fl_ctx,
+                f"Model after training: {weights}",
+            )
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            outgoing_dxo = DXO(data_kind=incoming_dxo.data_kind, data=weights, meta={})
+            return outgoing_dxo.to_shareable()
+        elif task_name == self._submit_model_task_name:
+            # Send the local model back to customer
+            weights = None
+            try:
+                weights = self._load_local_model(fl_ctx)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load model: {e}")
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            model_shareable = Shareable()
+            if weights:
+                incoming_dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights)
+                model_shareable = incoming_dxo.to_shareable()
+            else:
+                # Set return code.
+                model_shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+
+            return model_shareable
+        else:
+            # If unknown task name, set RC accordingly.
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+            return shareable
+
+    def _load_local_model(self, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+
+        model_load_path = os.path.join(model_path, self._model_name)
+        np_data = None
+        try:
+            np_data = np.load(model_load_path)
+        except Exception as e:
+            self.log_error(f"Unable to load local model: {np_data}")
+            return None
+
+        model = ModelLearnable()
+        model[NPConstants.NUMPY_KEY] = np_data
+
+        return model
+
+    def _save_local_model(self, fl_ctx: FLContext, model: dict):
+        # Save local model
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self._model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
+
+    def _get_exception_shareable(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/test_apps/sag_exception/config/config_fed_client.json b/test/app_testing/test_apps/sag_exception/config/config_fed_client.json
new file mode 100755
index 0000000000..6a42778034
--- /dev/null
+++ b/test/app_testing/test_apps/sag_exception/config/config_fed_client.json
@@ -0,0 +1,18 @@
+{
+  "format_version": 2,
+  "handlers": [],
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "np_trainer.NPTrainer",
+        "args": {}
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/sag_exception/config/config_fed_server.json b/test/app_testing/test_apps/sag_exception/config/config_fed_server.json
new file mode 100755
index 0000000000..79b48d85c6
--- /dev/null
+++ b/test/app_testing/test_apps/sag_exception/config/config_fed_server.json
@@ -0,0 +1,45 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "handlers": [],
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    }
+  ],
+  "workflows": [
+    {
+      "id": "scatter_and_gather",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 6000
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/sag_exception/custom/__init__.py b/test/app_testing/test_apps/sag_exception/custom/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/app_testing/test_apps/sag_exception/custom/constants.py b/test/app_testing/test_apps/sag_exception/custom/constants.py
new file mode 100755
index 0000000000..7e0e5dc276
--- /dev/null
+++ b/test/app_testing/test_apps/sag_exception/custom/constants.py
@@ -0,0 +1,2 @@
+class NPConstants:
+    NUMPY_KEY = "numpy_key"
diff --git a/test/app_testing/test_apps/sag_exception/custom/np_model_persistor.py b/test/app_testing/test_apps/sag_exception/custom/np_model_persistor.py
new file mode 100755
index 0000000000..c8f16e7121
--- /dev/null
+++ b/test/app_testing/test_apps/sag_exception/custom/np_model_persistor.py
@@ -0,0 +1,62 @@
+import os
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable, make_model_learnable, ModelLearnableKey
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPModelPersistor(ModelPersistor):
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_name = model_name
+        self.default_data = np.array(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32
+        )
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        # Get start round
+        start_round = fl_ctx.get_prop(AppConstants.START_ROUND, 0)
+
+        # Create a new numpy model
+        if start_round > 0:
+            try:
+                engine = fl_ctx.get_engine()
+                run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+                run_dir = engine.get_workspace().get_run_dir(run_number)
+                model_path = os.path.join(run_dir, self.model_dir, self.model_name)
+                data = np.load(model_path)
+            except Exception as e:
+                self.log_info(
+                    fl_ctx,
+                    f"Unable to load model from {model_path}. Using default data instead."
+                )
+                data = self.default_data.copy()
+        else:
+            data = self.default_data.copy()
+
+        weights = {NPConstants.NUMPY_KEY: data}
+        model_learnable = make_model_learnable(weights, {})
+
+        self.logger.info(f"Loaded initial model: {model_learnable[ModelLearnableKey.WEIGHTS]}")
+        return model_learnable
+
+    def save_model(self, model: ModelLearnable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self.model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
diff --git a/test/app_testing/test_apps/sag_exception/custom/np_trainer.py b/test/app_testing/test_apps/sag_exception/custom/np_trainer.py
new file mode 100755
index 0000000000..3a97f832bf
--- /dev/null
+++ b/test/app_testing/test_apps/sag_exception/custom/np_trainer.py
@@ -0,0 +1,73 @@
+import time
+
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPTrainer(Executor):
+    def __init__(
+        self,
+        delta=1,
+        sleep_time=0,
+        train_task_name=AppConstants.TASK_TRAIN,
+        submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL,
+        model_name="best_numpy.npy",
+        model_dir="model",
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        if not isinstance(delta, int):
+            raise TypeError("")
+
+        self._delta = delta
+        self._model_name = model_name
+        self._model_dir = model_dir
+        self._sleep_time = sleep_time
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     # Create all major components here.
+        #     pass
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        #     pass
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any kind of tasks waiting should check abort_signal regularly
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+            time.sleep(interval)
+            count += interval
+
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
+
+    def _get_exception_shareable(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/test_apps/sag_weights_diff/config/config_fed_client.json b/test/app_testing/test_apps/sag_weights_diff/config/config_fed_client.json
new file mode 100755
index 0000000000..6a42778034
--- /dev/null
+++ b/test/app_testing/test_apps/sag_weights_diff/config/config_fed_client.json
@@ -0,0 +1,18 @@
+{
+  "format_version": 2,
+  "handlers": [],
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "np_trainer.NPTrainer",
+        "args": {}
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/sag_weights_diff/config/config_fed_server.json b/test/app_testing/test_apps/sag_weights_diff/config/config_fed_server.json
new file mode 100755
index 0000000000..cb6797dbc0
--- /dev/null
+++ b/test/app_testing/test_apps/sag_weights_diff/config/config_fed_server.json
@@ -0,0 +1,45 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "handlers": [],
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.intime_accumulate_model_aggregator.InTimeAccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHT_DIFF"
+      }
+    }
+  ],
+  "workflows": [
+    {
+      "id": "scatter_and_gather",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 6000
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/sag_weights_diff/custom/__init__.py b/test/app_testing/test_apps/sag_weights_diff/custom/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/app_testing/test_apps/sag_weights_diff/custom/constants.py b/test/app_testing/test_apps/sag_weights_diff/custom/constants.py
new file mode 100755
index 0000000000..7e0e5dc276
--- /dev/null
+++ b/test/app_testing/test_apps/sag_weights_diff/custom/constants.py
@@ -0,0 +1,2 @@
+class NPConstants:
+    NUMPY_KEY = "numpy_key"
diff --git a/test/app_testing/test_apps/sag_weights_diff/custom/np_model_persistor.py b/test/app_testing/test_apps/sag_weights_diff/custom/np_model_persistor.py
new file mode 100755
index 0000000000..c8f16e7121
--- /dev/null
+++ b/test/app_testing/test_apps/sag_weights_diff/custom/np_model_persistor.py
@@ -0,0 +1,62 @@
+import os
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable, make_model_learnable, ModelLearnableKey
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPModelPersistor(ModelPersistor):
+    def __init__(self, model_dir="models", model_name="server.npy"):
+        super().__init__()
+
+        self.model_dir = model_dir
+        self.model_name = model_name
+        self.default_data = np.array(
+            [[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32
+        )
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        # Get start round
+        start_round = fl_ctx.get_prop(AppConstants.START_ROUND, 0)
+
+        # Create a new numpy model
+        if start_round > 0:
+            try:
+                engine = fl_ctx.get_engine()
+                run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+                run_dir = engine.get_workspace().get_run_dir(run_number)
+                model_path = os.path.join(run_dir, self.model_dir, self.model_name)
+                data = np.load(model_path)
+            except Exception as e:
+                self.log_info(
+                    fl_ctx,
+                    f"Unable to load model from {model_path}. Using default data instead."
+                )
+                data = self.default_data.copy()
+        else:
+            data = self.default_data.copy()
+
+        weights = {NPConstants.NUMPY_KEY: data}
+        model_learnable = make_model_learnable(weights, {})
+
+        self.logger.info(f"Loaded initial model: {model_learnable[ModelLearnableKey.WEIGHTS]}")
+        return model_learnable
+
+    def save_model(self, model: ModelLearnable, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self.model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self.model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
diff --git a/test/app_testing/test_apps/sag_weights_diff/custom/np_trainer.py b/test/app_testing/test_apps/sag_weights_diff/custom/np_trainer.py
new file mode 100755
index 0000000000..042f5a1dab
--- /dev/null
+++ b/test/app_testing/test_apps/sag_weights_diff/custom/np_trainer.py
@@ -0,0 +1,199 @@
+import os
+import time
+
+import numpy as np
+
+from constants import NPConstants
+from nvflare.apis.dxo import DXO, from_shareable, DataKind
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.app_constant import AppConstants
+
+
+class NPTrainer(Executor):
+    def __init__(
+        self,
+        delta=1,
+        sleep_time=0,
+        train_task_name=AppConstants.TASK_TRAIN,
+        submit_model_task_name=AppConstants.TASK_SUBMIT_MODEL,
+        model_name="best_numpy.npy",
+        model_dir="model",
+    ):
+        # Init functions of components should be very minimal. Init
+        # is called when json is read. A big init will cause json loading to halt
+        # for long time.
+        super().__init__()
+
+        if not isinstance(delta, int):
+            raise TypeError("")
+
+        self._delta = delta
+        self._model_name = model_name
+        self._model_dir = model_dir
+        self._sleep_time = sleep_time
+        self._train_task_name = train_task_name
+        self._submit_model_task_name = submit_model_task_name
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        # if event_type == EventType.START_RUN:
+        #     # Create all major components here.
+        #     pass
+        # elif event_type == EventType.END_RUN:
+        #     # Clean up resources (closing files, joining threads, removing dirs etc)
+        #     pass
+        pass
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        # Any kind of tasks waiting should check abort_signal regularly
+        count, interval = 0, 0.5
+        while count < self._sleep_time:
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+            time.sleep(interval)
+            count += interval
+
+        if task_name == self._train_task_name:
+
+            try:
+                incoming_dxo = from_shareable(shareable)
+            except BaseException as e:
+                self.system_panic(f"Unable to convert shareable to model definition. Exception {e}", fl_ctx)
+                return self._get_exception_shareable()
+
+            current_round = shareable.get_header(AppConstants.CURRENT_ROUND, None)
+            total_rounds = shareable.get_header(AppConstants.NUM_ROUNDS, None)
+
+            # Get data from numpy shareable
+            if incoming_dxo.data_kind != DataKind.WEIGHTS:
+                self.system_panic("Model dex should be of kind DataKind.WEIGHTS.", fl_ctx)
+                return self._get_exception_shareable()
+
+            weights = incoming_dxo.data
+
+            self.log_info(fl_ctx, f"Incoming data kind: {incoming_dxo.data_kind}")
+            self.log_info(fl_ctx, f"Model: \n{weights}")
+            self.log_info(fl_ctx, f"Current Round: {current_round}")
+            self.log_info(fl_ctx, f"Total Rounds: {total_rounds}")
+            self.log_info(fl_ctx, f"Task name: {task_name}")
+            self.log_info(fl_ctx, f"Client identity: {fl_ctx.get_identity_name()}")
+
+            # Check abort signal
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Doing some dummy training.
+            new_weights = {}
+            if weights:
+                if NPConstants.NUMPY_KEY in weights:
+                    new_weights[NPConstants.NUMPY_KEY] = weights[NPConstants.NUMPY_KEY] + self._delta
+                else:
+                    self.log_error(fl_ctx, "numpy_key not found in model.")
+                    shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+                    return shareable
+            else:
+                self.log_error(fl_ctx, "No model weights found in shareable.")
+                shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+                return shareable
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            # Save local model
+            try:
+                self._save_local_model(fl_ctx, weights)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Exception in saving local model: {e}.")
+
+            self.log_info(
+                fl_ctx,
+                f"Model after training: {weights}",
+            )
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            weights_diff = {k: new_weights[k]-weights[k] for k in weights.keys()}
+
+            outgoing_dxo = DXO(data_kind=DataKind.WEIGHT_DIFF, data=weights_diff, meta={})
+            return outgoing_dxo.to_shareable()
+        elif task_name == self._submit_model_task_name:
+            # Send the local model back to customer
+            weights = None
+            try:
+                weights = self._load_local_model(fl_ctx)
+            except Exception as e:
+                self.log_error(fl_ctx, f"Unable to load model: {e}")
+
+            if abort_signal.triggered:
+                return self._get_exception_shareable()
+
+            model_shareable = Shareable()
+            if weights:
+                incoming_dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights)
+                model_shareable = incoming_dxo.to_shareable()
+            else:
+                # Set return code.
+                model_shareable.set_return_code(ReturnCode.EXECUTION_RESULT_ERROR)
+
+            return model_shareable
+        else:
+            # If unknown task name, set RC accordingly.
+            shareable = Shareable()
+            shareable.set_return_code(ReturnCode.TASK_UNKNOWN)
+            return shareable
+
+    def _load_local_model(self, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+
+        model_load_path = os.path.join(model_path, self._model_name)
+        np_data = None
+        try:
+            np_data = np.load(model_load_path)
+        except Exception as e:
+            self.log_error(f"Unable to load local model: {np_data}")
+            return None
+
+        model = ModelLearnable()
+        model[NPConstants.NUMPY_KEY] = np_data
+
+        return model
+
+    def _save_local_model(self, fl_ctx: FLContext, model: dict):
+        # Save local model
+        engine = fl_ctx.get_engine()
+        run_number = fl_ctx.get_prop(FLContextKey.CURRENT_RUN)
+        run_dir = engine.get_workspace().get_run_dir(run_number)
+        model_path = os.path.join(run_dir, self._model_dir)
+        if not os.path.exists(model_path):
+            os.makedirs(model_path)
+
+        model_save_path = os.path.join(model_path, self._model_name)
+        if model_save_path:
+            with open(model_save_path, "wb") as f:
+                np.save(f, model[NPConstants.NUMPY_KEY])
+            self.log_info(fl_ctx, f"Saved numpy model to: {model_save_path}")
+
+    def _get_exception_shareable(self) -> Shareable:
+        """Abort execution. This is used if abort_signal is triggered. Users should
+        make sure they abort any running processes here.
+
+        Returns:
+            Shareable: Shareable with return_code.
+        """
+        shareable = Shareable()
+        shareable.set_return_code(ReturnCode.EXECUTION_EXCEPTION)
+        return shareable
diff --git a/test/app_testing/test_apps/tb_streaming/config/config_fed_client.json b/test/app_testing/test_apps/tb_streaming/config/config_fed_client.json
new file mode 100755
index 0000000000..62a41db7e0
--- /dev/null
+++ b/test/app_testing/test_apps/tb_streaming/config/config_fed_client.json
@@ -0,0 +1,40 @@
+{
+  "format_version": 2,
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "handlers": [],
+  "executors": [
+    {
+      "tasks": [
+        "poc"
+      ],
+      "executor": {
+        "id": "Executor",
+        "path": "custom_executor.CustomExecutor",
+        "args": {}
+      }
+    }
+  ],
+  "components": [
+    {
+      "id": "event_recorder",
+      "path": "nvflare.app_common.handlers.event_recorder.ClientEventRecorder",
+      "args": {}
+    },
+    {
+      "id": "analytic_sender",
+      "path": "nvflare.app_common.widgets.streaming.AnalyticsSender",
+      "args": {"event": "analytics"}
+    },
+    {
+      "id": "tb_analytics_receive",
+      "path": "nvflare.app_common.pt.tb_receiver.TBAnalyticsReceiver",
+      "args": {"events": ["analytics"]}
+    },
+    {
+      "id": "event_to_fed",
+      "path": "nvflare.app_common.widgets.convert_to_fed_event.ConvertToFedEvent",
+      "args": {"events_to_convert": ["analytics"], "fed_event_prefix": "fed."}
+    }
+  ]
+}
diff --git a/test/app_testing/test_apps/tb_streaming/config/config_fed_server.json b/test/app_testing/test_apps/tb_streaming/config/config_fed_server.json
new file mode 100755
index 0000000000..c67da26693
--- /dev/null
+++ b/test/app_testing/test_apps/tb_streaming/config/config_fed_server.json
@@ -0,0 +1,43 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "handlers": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "test.app_testing.np.np_model_persistor.NPModelPersistor",
+      "args": {}
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "event_recorder",
+      "path": "nvflare.app_common.handlers.event_recorder.ServerEventRecorder",
+      "args": {}
+    },
+    {
+      "id": "tb_analytics_receive",
+      "path": "nvflare.app_common.pt.tb_receiver.TBAnalyticsReceiver",
+      "args": {"events": ["fed.analytics"]}
+    }
+  ],
+  "workflows": [
+    {
+      "id": "CustomController",
+      "path": "custom_controller.CustomController",
+      "args": {
+        "min_clients": 2,
+        "num_rounds": 1,
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator"
+      }
+    }
+  ]
+}
diff --git a/test/app_testing/test_apps/tb_streaming/custom/__init__.py b/test/app_testing/test_apps/tb_streaming/custom/__init__.py
new file mode 100755
index 0000000000..e69de29bb2
diff --git a/test/app_testing/test_apps/tb_streaming/custom/custom_controller.py b/test/app_testing/test_apps/tb_streaming/custom/custom_controller.py
new file mode 100755
index 0000000000..0dbaee9d60
--- /dev/null
+++ b/test/app_testing/test_apps/tb_streaming/custom/custom_controller.py
@@ -0,0 +1,104 @@
+from nvflare.apis.client import Client
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.impl.controller import ClientTask, Controller, Task
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+from nvflare.app_common.abstract.learnable_persistor import LearnablePersistor
+from nvflare.app_common.abstract.shareable_generator import ShareableGenerator
+from nvflare.app_common.app_constant import AppConstants
+
+
+class CustomController(Controller):
+    def __init__(
+        self,
+        min_clients: int,
+        num_rounds: int,
+        persistor_id="persistor",
+        shareable_generator_id="shareable_generator",
+    ):
+        Controller.__init__(self)
+        self.persistor_id = persistor_id
+        self.shareable_generator_id = shareable_generator_id
+        self.persistor = None
+        self.shareable_gen = None
+
+        # config data
+        self._min_clients = min_clients
+        self._num_rounds = num_rounds
+
+        # workflow phases: init, train
+        self._phase = "init"
+        self._global_model = None
+
+    def start_controller(self, fl_ctx: FLContext):
+        self._phase = "init"
+        engine = fl_ctx.get_engine()
+
+        self.shareable_gen = engine.get_component(self.shareable_generator_id)
+        if not isinstance(self.shareable_gen, ShareableGenerator):
+            self.system_panic("shareable_gen should be an instance of ShareableGenerator.", fl_ctx)
+
+        self.persistor = engine.get_component(self.persistor_id)
+        if not isinstance(self.persistor, LearnablePersistor):
+            self.system_panic("persistor should be an instance of LearnablePersistor.", fl_ctx)
+
+        self._global_model = self.persistor.load(fl_ctx)
+        fl_ctx.set_prop(AppConstants.GLOBAL_MODEL, self._global_model, private=True, sticky=True)
+
+    def _prepare_training_ctx(self, client_task: ClientTask, fl_ctx: FLContext):
+        task = client_task.task
+        fl_ctx.set_prop("current_round", task.props["round"], private=False)
+        fl_ctx.set_prop("total_rounds", task.props["total"], private=False)
+
+    def _process_training_result(self, client_task: ClientTask, fl_ctx: FLContext):
+        task = client_task.task
+        task.data = client_task.result
+
+    def process_result_of_unknown_task(
+        self,
+        client: Client,
+        task_name: str,
+        client_task_id: str,
+        result: Shareable,
+        fl_ctx: FLContext,
+    ):
+        return None
+
+    def control_flow(self, abort_signal: Signal, fl_ctx: FLContext):
+        self._phase = "train"
+        engine = fl_ctx.get_engine()
+
+        # No rounds - will remove later
+        for r in range(self._num_rounds):
+            if not abort_signal:
+                return
+            task = Task(
+                name="poc",
+                data=self.shareable_gen.learnable_to_shareable(self._global_model, fl_ctx),
+                props={"round": r, "total": self._num_rounds},
+                timeout=0,
+                before_task_sent_cb=self._prepare_training_ctx,
+                result_received_cb=self._process_training_result,
+            )
+
+            client_list = engine.get_clients()
+            for c in client_list:
+                self.log_info(fl_ctx, f"@@@ client name {c.name}")
+
+            self.log_info(fl_ctx, f"@@@ Broadcast and wait {task.name}")
+            self.broadcast_and_wait(
+                task=task,
+                fl_ctx=fl_ctx,
+                targets=None,
+                min_responses=0,
+                abort_signal=abort_signal,
+            )
+            self.log_info(fl_ctx, f"@@@ Broadcast and wait - end {task.name}")
+
+            self._global_model = self.shareable_gen.shareable_to_learnable(task.data, fl_ctx)
+            self.persistor.save(self._global_model, fl_ctx)
+
+            self.logger.info("model saved")
+
+    def stop_controller(self, fl_ctx: FLContext):
+        self._phase = "finished"
diff --git a/test/app_testing/test_apps/tb_streaming/custom/custom_executor.py b/test/app_testing/test_apps/tb_streaming/custom/custom_executor.py
new file mode 100755
index 0000000000..00681bb482
--- /dev/null
+++ b/test/app_testing/test_apps/tb_streaming/custom/custom_executor.py
@@ -0,0 +1,45 @@
+import logging
+import random
+import time
+
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+
+
+class CustomExecutor(Executor):
+    def __init__(self, task_name: str = "poc", streamer_id: str = "analytic_sender"):
+        super().__init__()
+        if not isinstance(task_name, str):
+            raise TypeError("task name should be a string.")
+        if not isinstance(streamer_id, str):
+            raise TypeError("streamer_id should be a string.")
+        self.task_name = task_name
+        self.logger = logging.getLogger("POCExecutor")
+        self.streamer_id = streamer_id
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        if task_name in self.task_name:
+            peer_ctx = fl_ctx.get_prop(FLContextKey.PEER_CONTEXT)
+            r = peer_ctx.get_prop("current_round")
+
+            number = random.random()
+            engine = fl_ctx.get_engine()
+
+            # send analytics
+            analytic_sender = engine.get_component(self.streamer_id)
+            analytic_sender.write_scalar(fl_ctx, "random_number", number, global_step=r)
+            analytic_sender.write_text(fl_ctx, "debug_msg", "Hello world", global_step=r)
+            time.sleep(1.0)
+
+            return shareable
+        else:
+            raise ValueError(f'No such supported task "{task_name}". Implemented task name is {self.task_name}')
diff --git a/test/app_testing/test_apps/test_simple.yml b/test/app_testing/test_apps/test_simple.yml
new file mode 100644
index 0000000000..7c0c3375b4
--- /dev/null
+++ b/test/app_testing/test_apps/test_simple.yml
@@ -0,0 +1,57 @@
+
+
+
+tests:
+  - app_name: global_model_eval
+    validators:
+      - test.app_testing.test_apps.validators.global_model_eval_validator.GlobalModelEvalValidator
+      - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
+  - app_name: pt
+    validators:
+      - test.app_testing.test_apps.validators.pt_model_validator.PTModelValidator
+  - app_name: sag_weights_diff
+    validators:
+      - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
+  - app_name: cross_val_one_client
+    validators:
+      - test.app_testing.test_apps.validators.cross_val_single_client_validator.CrossValSingleClientValidator
+      - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
+  - app_name: cross_val
+    validators:
+      - test.app_testing.test_apps.validators.cross_val_result_validator.CrossResultValidator
+      - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
+  - app_name: sag
+    validators:
+      - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
+  - app_name: filters
+    validators:
+      - test.app_testing.test_apps.validators.filters_result_validator.FiltersResultValidator
+  - app_name: tb_streaming
+    validators:
+      - test.app_testing.test_apps.validators.tb_result_validator.TBResultValidator
+  - app_name: tf
+    validators:
+      - test.app_testing.test_apps.validators.tf_model_validator.TFModelValidator
+  - app_name: cyclic
+    validators:
+      - test.app_testing.test_apps.validators.tf_model_validator.TFModelValidator
+#  - app_name: sag_weights_diff
+#    validators:
+#      - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
+#  - app_name: cross_val_one_client
+#    validators:
+#      - test.app_testing.test_apps.validators.cross_val_single_client_validator.CrossValSingleClientValidator
+#      - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
+#  - app_name: cross_val
+#    validators:
+#      - test.app_testing.test_apps.validators.cross_val_result_validator.CrossResultValidator
+#      - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
+#  - app_name: sag
+#    validators:
+#      - test.app_testing.test_apps.validators.sag_result_validator.SAGResultValidator
+#  - app_name: filters
+#    validators:
+#      - test.app_testing.test_apps.validators.filters_result_validator.FiltersResultValidator
+#  - app_name: tb_streaming
+#    validators:
+#      - test.app_testing.test_apps.validators.tb_result_validator.TBResultValidator
diff --git a/test/app_testing/test_apps/tf/config/config_fed_client.json b/test/app_testing/test_apps/tf/config/config_fed_client.json
new file mode 100644
index 0000000000..d881eaad58
--- /dev/null
+++ b/test/app_testing/test_apps/tf/config/config_fed_client.json
@@ -0,0 +1,35 @@
+{
+  "format_version": 2,
+  "executors": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "executor": {
+        "path": "trainer.SimpleTrainer",
+        "args": {
+          "epochs_per_round": 2
+        }
+      }
+    }
+  ],
+  "task_result_filters": [
+    {
+      "tasks": [
+        "train"
+      ],
+      "filters": [
+        {
+          "path": "filter.ExcludeVars",
+          "args": {
+            "exclude_vars": [
+              "flatten"
+            ]
+          }
+        }
+      ]
+    }
+  ],
+  "task_data_filters": [],
+  "handlers": []
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/tf/config/config_fed_server.json b/test/app_testing/test_apps/tf/config/config_fed_server.json
new file mode 100644
index 0000000000..5a8dcba9b4
--- /dev/null
+++ b/test/app_testing/test_apps/tf/config/config_fed_server.json
@@ -0,0 +1,46 @@
+{
+  "format_version": 2,
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "persistor",
+      "path": "tf2_model_persistor.TF2ModelPersistor",
+      "args": {
+        "save_name": "tf2weights.pickle"
+      }
+    },
+    {
+      "id": "shareable_generator",
+      "path": "nvflare.app_common.shareablegenerators.full_model_shareable_generator.FullModelShareableGenerator",
+      "args": {}
+    },
+    {
+      "id": "aggregator",
+      "path": "nvflare.app_common.aggregators.accumulate_model_aggregator.AccumulateWeightedAggregator",
+      "args": {
+        "expected_data_kind": "WEIGHTS"
+      }
+    }
+  ],
+  "workflows": [
+    {
+      "id": "scatter_gather_ctl",
+      "path": "nvflare.app_common.workflows.scatter_and_gather.ScatterAndGather",
+      "args": {
+        "min_clients": 1,
+        "num_rounds": 3,
+        "start_round": 0,
+        "wait_time_after_min_received": 10,
+        "aggregator_id": "aggregator",
+        "persistor_id": "persistor",
+        "shareable_generator_id": "shareable_generator",
+        "train_task_name": "train",
+        "train_timeout": 0
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/app_testing/test_apps/tf/config/environment.json b/test/app_testing/test_apps/tf/config/environment.json
new file mode 100755
index 0000000000..0967ef424b
--- /dev/null
+++ b/test/app_testing/test_apps/tf/config/environment.json
@@ -0,0 +1 @@
+{}
diff --git a/test/app_testing/test_apps/tf/custom/__init__.py b/test/app_testing/test_apps/tf/custom/__init__.py
new file mode 100644
index 0000000000..3291348d45
--- /dev/null
+++ b/test/app_testing/test_apps/tf/custom/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/test/app_testing/test_apps/tf/custom/filter.py b/test/app_testing/test_apps/tf/custom/filter.py
new file mode 100644
index 0000000000..056a0265bc
--- /dev/null
+++ b/test/app_testing/test_apps/tf/custom/filter.py
@@ -0,0 +1,125 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import numpy as np
+from nvflare.apis.dxo import DXO, DataKind, from_shareable
+from nvflare.apis.filter import Filter
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+
+class ExcludeVars(Filter):
+    """
+        Exclude/Remove variables from Sharable
+
+    Args:
+        exclude_vars: if not specified (None), all layers are being encrypted;
+                      if list of variable/layer names, only specified variables are excluded;
+                      if string containing regular expression (e.g. "conv"), only matched variables are being excluded.
+    """
+
+    def __init__(self, exclude_vars=None):
+        super().__init__()
+        self.exclude_vars = exclude_vars
+        self.skip = False
+        if self.exclude_vars is not None:
+            if not (
+                isinstance(self.exclude_vars, list)
+                or isinstance(self.exclude_vars, str)
+            ):
+                self.skip = True
+                self.logger.debug(
+                    "Need to provide a list of layer names or a string for regex matching"
+                )
+                return
+
+            if isinstance(self.exclude_vars, list):
+                for var in self.exclude_vars:
+                    if not isinstance(var, str):
+                        self.skip = True
+                        self.logger.debug(
+                            "encrypt_layers needs to be a list of layer names to encrypt."
+                        )
+                        return
+                self.logger.debug(f"Excluding {self.exclude_vars} from shareable")
+            elif isinstance(self.exclude_vars, str):
+                self.exclude_vars = (
+                    re.compile(self.exclude_vars) if self.exclude_vars else None
+                )
+                if self.exclude_vars is None:
+                    self.skip = True
+                self.logger.debug(
+                    f'Excluding all layers based on regex matches with "{self.exclude_vars}"'
+                )
+        else:
+            self.logger.debug("Not excluding anything")
+            self.skip = True
+
+    def process(self, shareable: Shareable, fl_ctx: FLContext) -> Shareable:
+
+        self.log_debug(fl_ctx, "inside filter")
+        if self.skip:
+            return shareable
+
+        try:
+            dxo = from_shareable(shareable)
+        except:
+            self.log_exception(fl_ctx, "shareable data is not a valid DXO")
+            return shareable
+
+        assert isinstance(dxo, DXO)
+        if dxo.data_kind not in (DataKind.WEIGHT_DIFF, DataKind.WEIGHTS):
+            self.log_debug(fl_ctx, "I cannot handle {}".format(dxo.data_kind))
+            return shareable
+
+        if dxo.data is None:
+            self.log_debug(fl_ctx, "no data to filter")
+            return shareable
+
+        weights = dxo.data
+
+        # parse regex encrypt layers
+        if isinstance(self.exclude_vars, re.Pattern):
+            re_pattern = self.exclude_vars
+            self.exclude_vars = []
+            for var_name in weights.keys():
+                if re_pattern.search(var_name):
+                    self.exclude_vars.append(var_name)
+            self.log_debug(fl_ctx, f"Regex found {self.exclude_vars} matching layers.")
+            if len(self.exclude_vars) == 0:
+                self.log_warning(
+                    fl_ctx, f"No matching layers found with regex {re_pattern}"
+                )
+
+        # remove variables
+        n_excluded = 0
+        var_names = list(
+            weights.keys()
+        )  # needs to recast to list to be used in for loop
+        n_vars = len(var_names)
+        for var_name in var_names:
+            # self.logger.info(f"Checking {var_name}")
+            if var_name in self.exclude_vars:
+                self.log_debug(fl_ctx, f"Excluding {var_name}")
+                weights[var_name] = np.zeros(weights[var_name].shape)
+                n_excluded += 1
+        self.log_debug(
+            fl_ctx,
+            f"Excluded {n_excluded} of {n_vars} variables. {len(weights.keys())} remaining.",
+        )
+
+        dxo.data = weights
+        return dxo.update_shareable(shareable)
diff --git a/test/app_testing/test_apps/tf/custom/tf2_model_persistor.py b/test/app_testing/test_apps/tf/custom/tf2_model_persistor.py
new file mode 100644
index 0000000000..721dc1c746
--- /dev/null
+++ b/test/app_testing/test_apps/tf/custom/tf2_model_persistor.py
@@ -0,0 +1,107 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pickle
+import json
+
+import tensorflow as tf
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import ModelLearnable
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+from tf2_net import Net
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.abstract.model import make_model_learnable
+
+
+class TF2ModelPersistor(ModelPersistor):
+    def __init__(self, save_name="tf2_model.pkl"):
+        super().__init__()
+        self.save_name = save_name
+
+    def _initialize(self, fl_ctx: FLContext):
+        # get save path from FLContext
+        app_root = fl_ctx.get_prop(FLContextKey.APP_ROOT)
+        env_config = os.path.join(app_root, fl_ctx.get_prop(FLContextKey.ARGS).env)
+        with open(os.path.join(app_root, env_config)) as file:
+            env = json.load(file)
+
+        if env is not None:
+            if env.get("APP_CKPT_DIR", None):
+                fl_ctx.set_prop(
+                    AppConstants.LOG_DIR, env["APP_CKPT_DIR"], private=True, sticky=True
+                )
+            if env.get("APP_CKPT") is not None:
+                fl_ctx.set_prop(
+                    AppConstants.CKPT_PRELOAD_PATH,
+                    env["APP_CKPT"],
+                    private=True,
+                    sticky=True,
+                )
+
+        log_dir = fl_ctx.get_prop(AppConstants.LOG_DIR)
+        if log_dir:
+            self.log_dir = os.path.join(app_root, log_dir)
+        else:
+            self.log_dir = app_root
+        self._pkl_save_path = os.path.join(self.log_dir, self.save_name)
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
+
+    def load_model(self, fl_ctx: FLContext) -> ModelLearnable:
+        """
+            initialize and load the Model.
+
+        Args:
+            fl_ctx: FLContext
+
+        Returns:
+            Model object
+        """
+
+        if os.path.exists(self._pkl_save_path):
+            self.logger.info(f"Loading server weights")
+            with open(self._pkl_save_path, "rb") as f:
+                model_learnable = pickle.load(f)
+        else:
+            self.logger.info(f"Initializing server model")
+            network = Net()
+            loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+            network.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
+            _ = network(tf.keras.Input(shape=(28, 28)))
+            var_dict = {
+                network.get_layer(index=key).name: value
+                for key, value in enumerate(network.get_weights())
+            }
+            model_learnable = make_model_learnable(var_dict, dict())
+        return model_learnable
+
+    def handle_event(self, event: str, fl_ctx: FLContext):
+        if event == EventType.START_RUN:
+            self._initialize(fl_ctx)
+
+    def save_model(self, model_learnable: ModelLearnable, fl_ctx: FLContext):
+        """
+            persist the Model object
+
+        Args:
+            model: Model object
+            fl_ctx: FLContext
+        """
+        model_learnable_info = {k: str(type(v)) for k, v in model_learnable.items()}
+        self.logger.info(f"Saving aggregated server weights: \n {model_learnable_info}")
+        with open(self._pkl_save_path, "wb") as f:
+            pickle.dump(model_learnable, f)
diff --git a/test/app_testing/test_apps/tf/custom/tf2_net.py b/test/app_testing/test_apps/tf/custom/tf2_net.py
new file mode 100644
index 0000000000..526eeea0ca
--- /dev/null
+++ b/test/app_testing/test_apps/tf/custom/tf2_net.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tensorflow as tf
+
+
+class Net(tf.keras.Model):
+    def __init__(self):
+        super().__init__()
+        self.flatten = tf.keras.layers.Flatten(input_shape=(28, 28))
+        self.dense1 = tf.keras.layers.Dense(128, activation="relu")
+        self.dropout = tf.keras.layers.Dropout(0.2)
+        self.dense2 = tf.keras.layers.Dense(10)
+
+    def call(self, x):
+        x = self.flatten(x)
+        x = self.dense1(x)
+        x = self.dropout(x)
+        x = self.dense2(x)
+        return x
diff --git a/test/app_testing/test_apps/tf/custom/trainer.py b/test/app_testing/test_apps/tf/custom/trainer.py
new file mode 100644
index 0000000000..2355836207
--- /dev/null
+++ b/test/app_testing/test_apps/tf/custom/trainer.py
@@ -0,0 +1,142 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tensorflow as tf
+import numpy as np
+
+from nvflare.apis.dxo import DXO, DataKind, from_shareable
+from nvflare.apis.fl_constant import ReturnCode
+from nvflare.apis.event_type import EventType
+from nvflare.apis.executor import Executor
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable, make_reply
+from nvflare.apis.signal import Signal
+
+from tf2_net import Net
+
+
+class SimpleTrainer(Executor):
+    def __init__(self, epochs_per_round):
+        super().__init__()
+        self.epochs_per_round = epochs_per_round
+        self.train_images, self.train_labels = None, None
+        self.test_images, self.test_labels = None, None
+        self.model = None
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self.setup(fl_ctx)
+
+    def setup(self, fl_ctx: FLContext):
+        (self.train_images, self.train_labels), (
+            self.test_images,
+            self.test_labels,
+        ) = tf.keras.datasets.mnist.load_data()
+        self.train_images, self.test_images = (
+            self.train_images / 255.0,
+            self.test_images / 255.0,
+        )
+
+        # simulate separate datasets for each client by dividing MNIST dataset in half
+        client_name = fl_ctx.get_identity_name()
+        if client_name == "site-1":
+            self.train_images = self.train_images[: len(self.train_images) // 2]
+            self.train_labels = self.train_labels[: len(self.train_labels) // 2]
+            self.test_images = self.test_images[: len(self.test_images) // 2]
+            self.test_labels = self.test_labels[: len(self.test_labels) // 2]
+        elif client_name == "site-2":
+            self.train_images = self.train_images[len(self.train_images) // 2 :]
+            self.train_labels = self.train_labels[len(self.train_labels) // 2 :]
+            self.test_images = self.test_images[len(self.test_images) // 2 :]
+            self.test_labels = self.test_labels[len(self.test_labels) // 2 :]
+
+        model = Net()
+
+        loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+        model.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
+        _ = model(tf.keras.Input(shape=(28, 28)))
+        self.var_list = [
+            model.get_layer(index=index).name
+            for index in range(len(model.get_weights()))
+        ]
+        self.model = model
+
+    def execute(
+        self,
+        task_name: str,
+        shareable: Shareable,
+        fl_ctx: FLContext,
+        abort_signal: Signal,
+    ) -> Shareable:
+        """
+        This function is an extended function from the super class.
+        As a supervised learning based trainer, the train function will run
+        evaluate and train engines based on model weights from `shareable`.
+        After finishing training, a new `Shareable` object will be submitted
+        to server for aggregation.
+
+        Args:
+            task_name: dispatched task
+            shareable: the `Shareable` object acheived from server.
+            fl_ctx: the `FLContext` object achieved from server.
+            abort_signal: if triggered, the training will be aborted.
+
+        Returns:
+            a new `Shareable` object to be submitted to server for aggregation.
+        """
+
+        # retrieve model weights download from server's shareable
+        if abort_signal.triggered:
+            return make_reply(ReturnCode.OK)
+
+        if task_name != "train":
+            return shareable
+
+        dxo = from_shareable(shareable)
+        model_weights = dxo.data
+
+        # use previous round's client weights to replace excluded layers from server
+        prev_weights = {
+            self.model.get_layer(index=key).name: value
+            for key, value in enumerate(self.model.get_weights())
+        }
+        print("dxo")
+        ordered_model_weights = {key: model_weights.get(key) for key in prev_weights}
+        for key in self.var_list:
+            value = ordered_model_weights.get(key)
+            if np.all(value == 0):
+                ordered_model_weights[key] = prev_weights[key]
+
+        # update local model weights with received weights
+        self.model.set_weights(list(ordered_model_weights.values()))
+
+        # adjust LR or other training time info as needed
+        # such as callback in the fit function
+        self.model.fit(
+            self.train_images,
+            self.train_labels,
+            epochs=self.epochs_per_round,
+            validation_data=(self.test_images, self.test_labels),
+        )
+
+        # report updated weights in shareable
+        weights = {
+            self.model.get_layer(index=key).name: value
+            for key, value in enumerate(self.model.get_weights())
+        }
+        dxo = DXO(data_kind=DataKind.WEIGHTS, data=weights)
+
+        self.log_info(fl_ctx, "Local epochs finished. Returning shareable")
+
+        return dxo.update_shareable(shareable)
diff --git a/test/app_testing/test_apps/tf/resources/log.config b/test/app_testing/test_apps/tf/resources/log.config
new file mode 100644
index 0000000000..6b9761569b
--- /dev/null
+++ b/test/app_testing/test_apps/tf/resources/log.config
@@ -0,0 +1,27 @@
+[loggers]
+keys=root,modelLogger
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=fullFormatter
+
+[logger_root]
+level=INFO
+handlers=consoleHandler
+
+[logger_modelLogger]
+level=DEBUG
+handlers=consoleHandler
+qualname=modelLogger
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=fullFormatter
+args=(sys.stdout,)
+
+[formatter_fullFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
diff --git a/test/app_testing/test_apps/validators/cross_val_result_validator.py b/test/app_testing/test_apps/validators/cross_val_result_validator.py
new file mode 100644
index 0000000000..2a8750a1c9
--- /dev/null
+++ b/test/app_testing/test_apps/validators/cross_val_result_validator.py
@@ -0,0 +1,88 @@
+import os
+from test.app_testing.app_result_validator import AppResultValidator
+
+
+def check_cross_validation_result(server_data, client_data, run_data):
+
+    run_number = run_data["run_number"]
+    server_dir = server_data["server_path"]
+    client_names = list(client_data["client_names"])
+
+    server_run_dir = os.path.join(server_dir, "run_" + str(run_number))
+
+    if not os.path.exists(server_run_dir):
+        print(
+            f"check_cross_validation_result: server run dir {server_run_dir} doesn't exist."
+        )
+        return False
+
+    cross_val_dir = os.path.join(server_run_dir, "cross_site_val")
+    if not os.path.exists(cross_val_dir):
+        print(
+            f"check_cross_validation_result: models dir {cross_val_dir} doesn't exist."
+        )
+        return False
+
+    model_shareable_dir = os.path.join(cross_val_dir, "model_shareables")
+    if not os.path.exists(model_shareable_dir):
+        print(
+            f"check_cross_validation_result: model shareable directory {model_shareable_dir} doesn't exist."
+        )
+        return False
+
+    result_shareable_dir = os.path.join(cross_val_dir, "result_shareables")
+    if not os.path.exists(result_shareable_dir):
+        print(
+            f"check_cross_validation_result: result shareable directory {result_shareable_dir} doesn't exist."
+        )
+        return False
+
+    # There should be three files in model_shareable
+    server_model_names = ["SRV_" + server_data["server_name"]]
+    model_file_names = server_model_names.copy()
+    model_file_names = model_file_names + client_names
+    print(f"Model files to look for: {model_file_names}")
+
+    for model_file_name in model_file_names:
+        model_file = os.path.join(model_shareable_dir, model_file_name)
+        if not os.path.exists(model_file):
+            print(
+                f"check_cross_validation_result: model {model_file} doesn't exist in model shareable directory."
+            )
+            return False
+
+    # Check all the results
+    # results_file_names = ["client_1_server", "client_0_server", "client_1_client_0", "client_1_client_1",
+    #                       "client_0_client_1", "client_0_client_0"]
+    results_file_names = [f"{x}_{y}" for x in client_names for y in server_model_names]
+    for client_name in client_names:
+        results_file_names += [f"{client_name}_{x}" for x in client_names]
+    print(f"Result files to look for: {results_file_names}")
+
+    for results_file_name in results_file_names:
+        result_file = os.path.join(result_shareable_dir, results_file_name)
+        if not os.path.exists(result_file):
+            print(
+                f"check_cross_validation_result: result {result_file} doesn't exist in result shareable directory."
+            )
+            return False
+
+    return True
+
+
+class CrossResultValidator(AppResultValidator):
+    def __init__(self):
+        super(CrossResultValidator, self).__init__()
+
+    def validate_results(self, server_data, client_data, run_data) -> bool:
+
+        cross_val_result = check_cross_validation_result(
+            server_data, client_data, run_data
+        )
+
+        print(f"CrossVal Result: {cross_val_result}")
+
+        if not cross_val_result:
+            raise ValueError("Cross val failed.")
+
+        return cross_val_result
diff --git a/test/app_testing/test_apps/validators/cross_val_single_client_validator.py b/test/app_testing/test_apps/validators/cross_val_single_client_validator.py
new file mode 100644
index 0000000000..e819bfe92b
--- /dev/null
+++ b/test/app_testing/test_apps/validators/cross_val_single_client_validator.py
@@ -0,0 +1,89 @@
+import os
+
+from test.app_testing.app_result_validator import AppResultValidator
+
+
+def check_cross_validation_result(server_data, client_data, run_data):
+
+    run_number = run_data["run_number"]
+    server_dir = server_data["server_path"]
+    client_names = ["client_0"]
+
+    server_run_dir = os.path.join(server_dir, "run_" + str(run_number))
+
+    if not os.path.exists(server_run_dir):
+        print(
+            f"check_cross_validation_result: server run dir {server_run_dir} doesn't exist."
+        )
+        return False
+
+    cross_val_dir = os.path.join(server_run_dir, "cross_site_val")
+    if not os.path.exists(cross_val_dir):
+        print(
+            f"check_cross_validation_result: models dir {cross_val_dir} doesn't exist."
+        )
+        return False
+
+    model_shareable_dir = os.path.join(cross_val_dir, "model_shareables")
+    if not os.path.exists(model_shareable_dir):
+        print(
+            f"check_cross_validation_result: model shareable directory {model_shareable_dir} doesn't exist."
+        )
+        return False
+
+    result_shareable_dir = os.path.join(cross_val_dir, "result_shareables")
+    if not os.path.exists(result_shareable_dir):
+        print(
+            f"check_cross_validation_result: result shareable directory {result_shareable_dir} doesn't exist."
+        )
+        return False
+
+    # There should be three files in model_shareable
+    server_model_names = ["SRV_" + server_data["server_name"]]
+    model_file_names = server_model_names.copy()
+    model_file_names = model_file_names + client_names
+    print(f"Model files to look for: {model_file_names}")
+
+    for model_file_name in model_file_names:
+        model_file = os.path.join(model_shareable_dir, model_file_name)
+        if not os.path.exists(model_file):
+            print(
+                f"check_cross_validation_result: model {model_file} doesn't exist in model shareable directory."
+            )
+            return False
+
+    # Check all the results
+    # results_file_names = ["client_1_server", "client_0_server", "client_1_client_0", "client_1_client_1",
+    #                       "client_0_client_1", "client_0_client_0"]
+    results_file_names = [f"{x}_{y}" for x in client_names for y in server_model_names]
+    for client_name in client_names:
+        results_file_names += [f"{client_name}_{x}" for x in client_names]
+    print(f"Result files to look for: {results_file_names}")
+
+    for results_file_name in results_file_names:
+        result_file = os.path.join(result_shareable_dir, results_file_name)
+        if not os.path.exists(result_file):
+            print(
+                f"check_cross_validation_result: result {result_file} doesn't exist in result shareable directory."
+            )
+            return False
+
+    return True
+
+
+class CrossValSingleClientValidator(AppResultValidator):
+    def __init__(self):
+        super(CrossValSingleClientValidator, self).__init__()
+
+    def validate_results(self, server_data, client_data, run_data) -> bool:
+
+        cross_val_result = check_cross_validation_result(
+            server_data, client_data, run_data
+        )
+
+        print(f"CrossVal Result: {cross_val_result}")
+
+        if not cross_val_result:
+            raise ValueError("Cross val failed.")
+
+        return cross_val_result
diff --git a/test/app_testing/test_apps/validators/filters_result_validator.py b/test/app_testing/test_apps/validators/filters_result_validator.py
new file mode 100644
index 0000000000..4df016c4a3
--- /dev/null
+++ b/test/app_testing/test_apps/validators/filters_result_validator.py
@@ -0,0 +1,37 @@
+import os
+from test.app_testing.app_result_validator import AppResultValidator
+
+import numpy as np
+
+
+class FiltersResultValidator(AppResultValidator):
+    def validate_results(self, server_data, client_data, run_data) -> bool:
+        run_number = run_data["run_number"]
+        server_dir = server_data["server_path"]
+
+        server_run_dir = os.path.join(server_dir, f"run_{run_number}")
+
+        if not os.path.exists(server_run_dir):
+            print(
+                f"FiltersResultValidator: server run dir {server_run_dir} doesn't exist."
+            )
+            return False
+
+        models_dir = os.path.join(server_run_dir, "models")
+        if not os.path.exists(models_dir):
+            print(f"FiltersResultValidator: models dir {models_dir} doesn't exist.")
+            return False
+
+        model_path = os.path.join(models_dir, "server.npy")
+        if not os.path.isfile(model_path):
+            print(f"FiltersResultValidator: model_path {model_path} doesn't exist.")
+            return False
+
+        try:
+            data = np.load(model_path)
+            print(f"FiltersResultValidator: Data loaded: {data}.")
+        except Exception as e:
+            print(f"Exception in FiltersResultValidator: {e.__str__()}")
+            return False
+
+        return True
diff --git a/test/app_testing/test_apps/validators/global_model_eval_validator.py b/test/app_testing/test_apps/validators/global_model_eval_validator.py
new file mode 100644
index 0000000000..85fde95420
--- /dev/null
+++ b/test/app_testing/test_apps/validators/global_model_eval_validator.py
@@ -0,0 +1,85 @@
+import os
+from test.app_testing.app_result_validator import AppResultValidator
+
+
+def check_global_model_eval_result(server_data, client_data, run_data):
+
+    run_number = run_data["run_number"]
+    server_dir = server_data["server_path"]
+    client_names = list(client_data["client_names"])
+
+    server_run_dir = os.path.join(server_dir, "run_" + str(run_number))
+
+    if not os.path.exists(server_run_dir):
+        print(
+            f"check_global_model_eval_result: server run dir {server_run_dir} doesn't exist."
+        )
+        return False
+
+    cross_val_dir = os.path.join(server_run_dir, "cross_site_val")
+    if not os.path.exists(cross_val_dir):
+        print(
+            f"check_global_model_eval_result: models dir {cross_val_dir} doesn't exist."
+        )
+        return False
+
+    model_shareable_dir = os.path.join(cross_val_dir, "model_shareables")
+    if not os.path.exists(model_shareable_dir):
+        print(
+            f"check_global_model_eval_result: model shareable directory {model_shareable_dir} doesn't exist."
+        )
+        return False
+
+    result_shareable_dir = os.path.join(cross_val_dir, "result_shareables")
+    if not os.path.exists(result_shareable_dir):
+        print(
+            f"check_global_model_eval_result: result shareable directory {result_shareable_dir} doesn't exist."
+        )
+        return False
+
+    # There should be three files in model_shareable
+    server_model_names = ["SRV_" + server_data["server_name"]]
+    model_file_names = server_model_names.copy()
+    print(f"Model files to look for: {model_file_names}")
+
+    for model_file_name in model_file_names:
+        model_file = os.path.join(model_shareable_dir, model_file_name)
+        if not os.path.exists(model_file):
+            print(
+                f"check_global_model_eval_result: model {model_file} doesn't exist in model shareable directory."
+            )
+            return False
+
+    # Check all the results
+    # results_file_names = ["client_1_server", "client_0_server", "client_1_client_0", "client_1_client_1",
+    #                       "client_0_client_1", "client_0_client_0"]
+    results_file_names = [f"{x}_{y}" for x in client_names for y in server_model_names]
+    print(f"Result files to look for: {results_file_names}")
+
+    for results_file_name in results_file_names:
+        result_file = os.path.join(result_shareable_dir, results_file_name)
+        if not os.path.exists(result_file):
+            print(
+                f"check_global_model_eval_result: result {result_file} doesn't exist in result shareable directory."
+            )
+            return False
+
+    return True
+
+
+class GlobalModelEvalValidator(AppResultValidator):
+    def __init__(self):
+        super(GlobalModelEvalValidator, self).__init__()
+
+    def validate_results(self, server_data, client_data, run_data) -> bool:
+
+        cross_val_result = check_global_model_eval_result(
+            server_data, client_data, run_data
+        )
+
+        print(f"CrossVal Result: {cross_val_result}")
+
+        if not cross_val_result:
+            raise ValueError("Cross val failed.")
+
+        return cross_val_result
diff --git a/test/app_testing/test_apps/validators/pt_model_validator.py b/test/app_testing/test_apps/validators/pt_model_validator.py
new file mode 100644
index 0000000000..686e38a1ae
--- /dev/null
+++ b/test/app_testing/test_apps/validators/pt_model_validator.py
@@ -0,0 +1,45 @@
+import os
+from collections import OrderedDict
+
+import numpy as np
+
+from test.app_testing.app_result_validator import AppResultValidator
+
+
+def check_pt_results(server_data, client_data, run_data):
+    run_number = run_data["run_number"]
+    server_dir = server_data["server_path"]
+
+    server_run_dir = os.path.join(server_dir, "run_" + str(run_number))
+    intended_model = OrderedDict([('model',
+                                   OrderedDict([('linear.weight',
+                                                np.array([[-0.1234, -0.1452, -0.1570]])),
+                                                ('linear.bias', np.array([0.4643]))])),
+                                  ('train_conf', {'train': {'model': 'SimpleNetwork'}})])
+
+    if not os.path.exists(server_run_dir):
+        print(
+            f"check_sag_results: server run dir {server_run_dir} doesn't exist."
+        )
+        return False
+
+    models_dir = os.path.join(server_run_dir, "app_server")
+    if not os.path.exists(models_dir):
+        print(f"check_sag_results: models dir {models_dir} doesn't exist.")
+        return False
+
+    model_path = os.path.join(models_dir, "FL_global_model.pt")
+    if not os.path.isfile(model_path):
+        print(f"check_sag_results: model_path {model_path} doesn't exist.")
+        return False
+
+    return True
+
+
+class PTModelValidator(AppResultValidator):
+
+    def __init__(self):
+        super(PTModelValidator, self).__init__()
+
+    def validate_results(self, server_data, client_data, run_data) -> bool:
+        return check_pt_results(server_data, client_data, run_data)
\ No newline at end of file
diff --git a/test/app_testing/test_apps/validators/sag_result_validator.py b/test/app_testing/test_apps/validators/sag_result_validator.py
new file mode 100644
index 0000000000..65abce2124
--- /dev/null
+++ b/test/app_testing/test_apps/validators/sag_result_validator.py
@@ -0,0 +1,63 @@
+import os
+from test.app_testing.app_result_validator import AppResultValidator
+
+import numpy as np
+
+
+def check_sag_results(server_data, client_data, run_data):
+    run_number = run_data["run_number"]
+    server_dir = server_data["server_path"]
+
+    server_run_dir = os.path.join(server_dir, "run_" + str(run_number))
+    intended_model = np.array(
+        [[4., 5., 6.],
+         [7., 8., 9.],
+         [10., 11., 12.]],
+        dtype="float32",
+    )
+
+    if not os.path.exists(server_run_dir):
+        print(
+            f"check_sag_results: server run dir {server_run_dir} doesn't exist."
+        )
+        return False
+
+    models_dir = os.path.join(server_run_dir, "models")
+    if not os.path.exists(models_dir):
+        print(f"check_sag_results: models dir {models_dir} doesn't exist.")
+        return False
+
+    model_path = os.path.join(models_dir, "server.npy")
+    if not os.path.isfile(model_path):
+        print(f"check_sag_results: model_path {model_path} doesn't exist.")
+        return False
+
+    try:
+        data = np.load(model_path)
+        print(f"check_sag_result: Data loaded: {data}.")
+        np.testing.assert_equal(data, intended_model)
+    except Exception as e:
+        print(f"Exception in validating ScatterAndGather model: {e.__str__()}")
+        return False
+
+    return True
+
+
+class SAGResultValidator(AppResultValidator):
+    def __init__(self):
+        super(SAGResultValidator, self).__init__()
+
+    def validate_results(self, server_data, client_data, run_data) -> bool:
+        # pass
+        # print(f"CrossValValidator server data: {server_data}")
+        # print(f"CrossValValidator client data: {client_data}")
+        # print(f"CrossValValidator run data: {run_data}")
+
+        fed_avg_result = check_sag_results(server_data, client_data, run_data)
+
+        print(f"ScatterAndGather Result: {fed_avg_result}")
+
+        if not fed_avg_result:
+            raise ValueError("Scatter and gather failed.")
+
+        return fed_avg_result
diff --git a/test/app_testing/test_apps/validators/tb_result_validator.py b/test/app_testing/test_apps/validators/tb_result_validator.py
new file mode 100644
index 0000000000..66e2aa367c
--- /dev/null
+++ b/test/app_testing/test_apps/validators/tb_result_validator.py
@@ -0,0 +1,26 @@
+import os
+from test.app_testing.app_result_validator import AppResultValidator
+
+TB_PATH = "tb_events"
+
+
+class TBResultValidator(AppResultValidator):
+    def validate_results(self, server_data, client_data, run_data) -> bool:
+        run_number = run_data["run_number"]
+        server_path = server_data["server_path"]
+
+        server_run_dir = os.path.join(server_path, f"run_{run_number}")
+        server_tb_root_dir = os.path.join(server_run_dir, TB_PATH)
+        if not os.path.exists(server_tb_root_dir):
+            return False
+
+        for i, client_path in enumerate(client_data["client_paths"]):
+            client_run_dir = os.path.join(client_path, f"run_{run_number}")
+            client_side_client_tb_dir = os.path.join(client_run_dir, TB_PATH, client_data["client_names"][i])
+            if not os.path.exists(client_side_client_tb_dir):
+                return False
+
+            server_side_client_tb_dir = os.path.join(server_tb_root_dir, client_data["client_names"][i])
+            if not os.path.exists(server_side_client_tb_dir):
+                return False
+        return True
diff --git a/test/app_testing/test_apps/validators/tf_model_validator.py b/test/app_testing/test_apps/validators/tf_model_validator.py
new file mode 100644
index 0000000000..d1b4e2eb6b
--- /dev/null
+++ b/test/app_testing/test_apps/validators/tf_model_validator.py
@@ -0,0 +1,46 @@
+import os
+from collections import OrderedDict
+from test.app_testing.app_result_validator import AppResultValidator
+
+import pickle
+import numpy as np
+
+
+def check_tf_results(server_data, client_data, run_data):
+    run_number = run_data["run_number"]
+    server_dir = server_data["server_path"]
+
+    server_run_dir = os.path.join(server_dir, "run_" + str(run_number))
+
+    if not os.path.exists(server_run_dir):
+        print(f"check_tf_results: server run dir {server_run_dir} doesn't exist.")
+        return False
+
+    models_dir = os.path.join(server_run_dir, "app_server")
+    if not os.path.exists(models_dir):
+        print(f"check_sag_results: models dir {models_dir} doesn't exist.")
+        return False
+
+    model_path = os.path.join(models_dir, "tf2weights.pickle")
+    if not os.path.isfile(model_path):
+        print(f"check_tf_results: model_path {model_path} doesn't exist.")
+        return False
+
+    try:
+        data = pickle.load(open(model_path, "rb"))
+        print(f"check_tf_result: Data loaded: {data}.")
+        assert "weights" in data
+        assert "meta" in data
+    except Exception as e:
+        print(f"Exception in validating TF model: {e.__str__()}")
+        return False
+
+    return True
+
+
+class TFModelValidator(AppResultValidator):
+    def __init__(self):
+        super().__init__()
+
+    def validate_results(self, server_data, client_data, run_data) -> bool:
+        return check_tf_results(server_data, client_data, run_data)
diff --git a/test/app_testing/test_constants.py b/test/app_testing/test_constants.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/test/app_testing/test_runner.py b/test/app_testing/test_runner.py
new file mode 100644
index 0000000000..a82c20714e
--- /dev/null
+++ b/test/app_testing/test_runner.py
@@ -0,0 +1,136 @@
+import argparse
+import importlib
+import os
+import sys
+import time
+import traceback
+
+import yaml
+
+from test.app_testing.admin_controller import AdminController
+from test.app_testing.site_launcher import SiteLauncher
+
+
+def get_module_class_from_full_path(full_path):
+    tokens = full_path.split(".")
+    cls_name = tokens[-1]
+    mod_name = ".".join(tokens[: len(tokens) - 1])
+    return mod_name, cls_name
+
+
+def read_yaml(yaml_file_path):
+    if not os.path.exists(yaml_file_path):
+        print(f"Yaml file doesnt' exist at {yaml_file_path}")
+        return None
+
+    data = None
+    with open(yaml_file_path, "rb") as f:
+        data = yaml.safe_load(f)
+
+    return [(x["app_name"], x["validators"]) for x in data["tests"]]
+
+
+if __name__ == "__main__":
+    site_launcher = None
+    cleanup = True
+    try:
+        parser = argparse.ArgumentParser(description="Run NVFlare tests")
+        parser.add_argument("--poc", "-p", type=str, help="Poc Directory")
+        parser.add_argument("--n_clients", "-n", type=int, help="Number of clients.")
+        parser.add_argument(
+            "--app_path", "-ap", type=str, help="Directory for the apps."
+        )
+        parser.add_argument("--yaml", "-y", type=str, help="Yaml test config path.")
+        parser.add_argument(
+            "--cleanup", "-c", action="store_true", help="Whether to cleanup "
+        )
+
+        args = parser.parse_args()
+
+        cleanup = args.cleanup
+
+        test_apps = read_yaml(args.yaml)
+
+        print(f"{test_apps=}")
+
+        site_launcher = SiteLauncher(poc_directory=args.poc, app_path=args.app_path)
+
+        site_launcher.start_server()
+        site_launcher.start_clients(n=args.n_clients)
+
+        admin_controller = AdminController(app_path=args.app_path)
+        admin_controller.initialize()
+
+        admin_controller.ensure_clients_started(
+            num_clients=args.n_clients
+        )
+
+        print(f"Server status: {admin_controller.server_status()}.")
+
+        app_results = []
+        for app_data in test_apps:
+            start_time = time.time()
+
+            test_app, validators = app_data
+
+            print(f"Running app {test_app} with {validators}")
+
+            admin_controller.deploy_app(app_name=test_app)
+
+            # time.sleep(2)
+
+            print(f"Server status after app deployment: {admin_controller.server_status()}.")
+            print(f"Client status after app deployment: {admin_controller.client_status()}")
+
+            admin_controller.run_app()
+
+            server_data = site_launcher.get_server_data()
+            client_data = site_launcher.get_client_data()
+            run_data = admin_controller.get_run_data()
+
+            # Get the app validator
+            if validators:
+                validate_result = True
+                for validator_module in validators:
+
+                    # Create validator instance
+                    module_name, class_name = get_module_class_from_full_path(
+                        validator_module
+                    )
+                    app_validator_cls = getattr(
+                        importlib.import_module(module_name), class_name
+                    )
+                    app_validator = app_validator_cls()
+
+                    app_validate_res = app_validator.validate_results(
+                        server_data=server_data,
+                        client_data=client_data,
+                        run_data=run_data,
+                    )
+                    validate_result = validate_result and app_validate_res
+
+                app_results.append((test_app, validate_result))
+            else:
+                print(f"No validators provided so results can't be checked.")
+
+            print(f"Finished running {test_app} in {time.time()-start_time} seconds.")
+
+        print(f"App results: {app_results}")
+        failure = False
+        for app_name, app_result in app_results:
+            print(f"App name: {app_name}, Result: {app_result}")
+            if not app_result:
+                failure = True
+
+        if failure:
+            sys.exit(1)
+    except BaseException as e:
+        traceback.print_exc()
+        print(f"Exception in test run: {e.__str__()}")
+        raise ValueError("Tests failed") from e
+    finally:
+        if site_launcher:
+            site_launcher.stop_all_sites()
+
+            if cleanup:
+                site_launcher.finalize()
diff --git a/test/controller/__init__.py b/test/controller/__init__.py
new file mode 100644
index 0000000000..917ae28897
--- /dev/null
+++ b/test/controller/__init__.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/test/controller/controller_test.py b/test/controller/controller_test.py
new file mode 100644
index 0000000000..f0d284b614
--- /dev/null
+++ b/test/controller/controller_test.py
@@ -0,0 +1,128 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+import uuid
+
+from nvflare.apis.client import Client
+from nvflare.apis.controller_spec import Task
+from nvflare.apis.fl_context import FLContext, FLContextManager
+from nvflare.apis.impl.controller import Controller
+from nvflare.apis.shareable import Shareable
+from nvflare.apis.signal import Signal
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+
+
+def create_task(name, data=None, timeout=0, before_task_sent_cb=None, result_received_cb=None, task_done_cb=None):
+    data = Shareable() if data is None else data
+    task = Task(
+        name=name,
+        data=data,
+        timeout=timeout,
+        before_task_sent_cb=before_task_sent_cb,
+        result_received_cb=result_received_cb,
+        task_done_cb=task_done_cb,
+    )
+    return task
+
+
+def create_client(name, token=None):
+    token = str(uuid.uuid4()) if token is None else token
+    return Client(name=name, token=token)
+
+
+# - provide a easy way for researchers to test their own Controller / their own control loop?
+# - how can they write their own test cases, simulating different client in diff. scenario...
+
+
+class DummyController(Controller):
+    def __init__(self):
+        super().__init__(task_check_period=0.1)
+
+    def control_flow(self, abort_signal: Signal, fl_ctx: FLContext):
+        print(f"Entering control loop of TestController")
+
+    def start_controller(self, fl_ctx: FLContext):
+        print("Start controller")
+
+    def stop_controller(self, fl_ctx: FLContext):
+        print("Stop controller")
+
+    def process_result_of_unknown_task(
+        self, client: Client, task_name: str, client_task_id: str, result: Shareable, fl_ctx: FLContext
+    ):
+        raise RuntimeError(f"Unknown task: {task_name} from client {client.name}.")
+
+
+class MockEngine:
+    def __init__(self, run_num=0):
+        self.fl_ctx_mgr = FLContextManager(
+            engine=self,
+            identity_name="__mock_engine",
+            run_num=run_num,
+            public_stickers={},
+            private_stickers={},
+        )
+
+    def new_context(self):
+        return self.fl_ctx_mgr.new_context()
+
+    def fire_event(self, event_type: str, fl_ctx: FLContext):
+        pass
+
+
+def launch_task(controller, method, task, fl_ctx, kwargs):
+    if method == "broadcast":
+        controller.broadcast(task=task, fl_ctx=fl_ctx, **kwargs)
+    elif method == "broadcast_and_wait":
+        controller.broadcast_and_wait(task=task, fl_ctx=fl_ctx, **kwargs)
+    elif method == "send":
+        controller.send(task=task, fl_ctx=fl_ctx, **kwargs)
+    elif method == "send_and_wait":
+        controller.send_and_wait(task=task, fl_ctx=fl_ctx, **kwargs)
+    elif method == "relay":
+        controller.relay(task=task, fl_ctx=fl_ctx, **kwargs)
+    elif method == "relay_and_wait":
+        controller.relay_and_wait(task=task, fl_ctx=fl_ctx, **kwargs)
+
+
+def get_ready(thread, sleep_time=0.1):
+    thread.start()
+    time.sleep(sleep_time)
+
+
+def get_controller_and_engine():
+    return DummyController(), MockEngine()
+
+
+class TestController:
+    NO_RELAY = ["broadcast", "broadcast_and_wait", "send", "send_and_wait"]
+    RELAY = ["relay", "relay_and_wait"]
+    ALL_APIS = NO_RELAY + RELAY
+
+    @staticmethod
+    def start_controller():
+        """starts the controller"""
+        controller, engine = get_controller_and_engine()
+        fl_ctx = engine.fl_ctx_mgr.new_context()
+        controller.initialize_run(fl_ctx=fl_ctx)
+        return controller, fl_ctx
+
+    @staticmethod
+    def stop_controller(controller, fl_ctx):
+        """stops the controller"""
+        controller.finalize_run(fl_ctx=fl_ctx)
diff --git a/test/controller/test_basic.py b/test/controller/test_basic.py
new file mode 100644
index 0000000000..1a57825fd4
--- /dev/null
+++ b/test/controller/test_basic.py
@@ -0,0 +1,185 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+import uuid
+
+import pytest
+
+from nvflare.apis.controller_spec import TaskCompletionStatus
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+from .controller_test import TestController, create_client, create_task, get_ready, launch_task
+
+
+class TestBasic(TestController):
+    @pytest.mark.parametrize("task_name,client_name", [["__test_task", "__test_client"]])
+    def test_process_submission_invalid_task(self, task_name, client_name):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+        with pytest.raises(RuntimeError, match=f"Unknown task: {task_name} from client {client_name}."):
+            controller.process_submission(
+                client=client, task_name=task_name, task_id=str(uuid.uuid4()), fl_ctx=FLContext(), result=Shareable()
+            )
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    @pytest.mark.parametrize("num_client_requests", [1, 2, 3, 4])
+    def test_process_task_request_client_request_multiple_times(self, method, num_client_requests):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        get_ready(launch_thread)
+
+        for i in range(num_client_requests):
+            task_name_out, _, data = controller.process_task_request(client, fl_ctx)
+            assert task_name_out == "__test_task"
+            assert data == input_data
+        assert task.last_client_task_map["__test_client"].task_send_count == num_client_requests
+        controller.cancel_task(task)
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    def test_process_submission(self, method):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+        task = create_task("__test_task")
+        kwargs = {"targets": [client]}
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": kwargs,
+            },
+        )
+        get_ready(launch_thread)
+
+        task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+        # in here we make up client results:
+        result = Shareable()
+        result["result"] = "result"
+
+        controller.process_submission(
+            client=client, task_name="__test_task", task_id=client_task_id, fl_ctx=fl_ctx, result=result
+        )
+        assert task.last_client_task_map["__test_client"].result == result
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    @pytest.mark.parametrize("timeout", [1, 2])
+    def test_task_timeout(self, method, timeout):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+        task = create_task(name="__test_task", data=Shareable(), timeout=timeout)
+
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        get_ready(launch_thread)
+
+        assert controller.get_num_standing_tasks() == 1
+        time.sleep(timeout + 1)
+        assert controller.get_num_standing_tasks() == 0
+        assert task.completion_status == TaskCompletionStatus.TIMEOUT
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    def test_cancel_task(self, method):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+        task = create_task(name="__test_task")
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        controller.cancel_task(task=task)
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 0
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    def test_cancel_all_tasks(self, method):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+        task = create_task("__test_task")
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        get_ready(launch_thread)
+        task1 = create_task("__test_task1")
+        launch_thread1 = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task1,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        get_ready(launch_thread1)
+        assert controller.get_num_standing_tasks() == 2
+
+        controller.cancel_all_tasks()
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 0
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        assert task1.completion_status == TaskCompletionStatus.CANCELLED
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
diff --git a/test/controller/test_broadcast_behavior.py b/test/controller/test_broadcast_behavior.py
new file mode 100644
index 0000000000..d5c2355341
--- /dev/null
+++ b/test/controller/test_broadcast_behavior.py
@@ -0,0 +1,256 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+
+import pytest
+
+from nvflare.apis.controller_spec import TaskCompletionStatus
+from nvflare.apis.shareable import Shareable
+
+from .controller_test import TestController, create_client, create_task, get_ready, launch_task
+
+
+@pytest.mark.parametrize("method", ["broadcast", "broadcast_and_wait"])
+class TestBroadcastBehavior(TestController):
+    @pytest.mark.parametrize("num_of_clients", [1, 2, 3, 4])
+    def test_client_receive_only_one_task(self, method, num_of_clients):
+        controller, fl_ctx = self.start_controller()
+        clients = [create_client(f"__test_client{i}") for i in range(num_of_clients)]
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": None, "min_responses": num_of_clients},
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        for client in clients:
+            task_name_out = ""
+            client_task_id = ""
+            data = None
+            while task_name_out == "":
+                task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+                time.sleep(0.1)
+            assert task_name_out == "__test_task"
+            assert data == input_data
+            assert task.last_client_task_map[client.name].task_send_count == 1
+            assert controller.get_num_standing_tasks() == 1
+            _, next_client_task_id, _ = controller.process_task_request(client, fl_ctx)
+            assert next_client_task_id == client_task_id
+            assert task.last_client_task_map[client.name].task_send_count == 2
+
+            result = Shareable()
+            result["result"] = "result"
+            controller.process_submission(
+                client=client,
+                task_name="__test_task",
+                task_id=client_task_id,
+                fl_ctx=fl_ctx,
+                result=result,
+            )
+            assert task.last_client_task_map[client.name].result == result
+
+        controller._check_tasks()
+        assert task.completion_status == TaskCompletionStatus.OK
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("num_of_clients", [1, 2, 3, 4])
+    def test_only_client_in_target_will_get_task(self, method, num_of_clients):
+        controller, fl_ctx = self.start_controller()
+        clients = [create_client(f"__test_client{i}") for i in range(num_of_clients)]
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [clients[0]], "min_responses": 0},
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        task_name_out = ""
+        data = None
+        while task_name_out == "":
+            task_name_out, client_task_id, data = controller.process_task_request(clients[0], fl_ctx)
+            time.sleep(0.1)
+        assert task_name_out == "__test_task"
+        assert data == input_data
+        assert task.last_client_task_map[clients[0].name].task_send_count == 1
+        assert controller.get_num_standing_tasks() == 1
+
+        for client in clients[1:]:
+            task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+            assert task_name_out == ""
+            assert client_task_id == ""
+
+        controller.cancel_task(task)
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("min_responses", [1, 2, 3, 4])
+    def test_task_only_exit_when_min_responses_received(self, method, min_responses):
+        controller, fl_ctx = self.start_controller()
+        clients = [create_client(f"__test_client{i}") for i in range(min_responses)]
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": None, "min_responses": min_responses},
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        client_task_ids = []
+        for client in clients:
+            task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+            client_task_ids.append(client_task_id)
+            assert task_name_out == "__test_task"
+
+        for client, client_task_id in zip(clients, client_task_ids):
+            result = Shareable()
+            controller._check_tasks()
+            assert controller.get_num_standing_tasks() == 1
+            controller.process_submission(
+                client=client, task_name="__test_task", task_id=client_task_id, result=result, fl_ctx=fl_ctx
+            )
+
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 0
+        assert task.completion_status == TaskCompletionStatus.OK
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("min_responses", [1, 2, 3, 4])
+    @pytest.mark.parametrize("wait_time_after_min_received", [1, 2])
+    def test_task_only_exit_when_wait_time_after_min_received(
+        self, method, min_responses, wait_time_after_min_received
+    ):
+        controller, fl_ctx = self.start_controller()
+        clients = [create_client(f"__test_client{i}") for i in range(min_responses)]
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {
+                    "targets": None,
+                    "min_responses": min_responses,
+                    "wait_time_after_min_received": wait_time_after_min_received,
+                },
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        client_task_ids = []
+        for client in clients:
+            task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+            client_task_ids.append(client_task_id)
+            assert task_name_out == "__test_task"
+
+        for client, client_task_id in zip(clients, client_task_ids):
+            result = Shareable()
+            controller.process_submission(
+                client=client, task_name="__test_task", task_id=client_task_id, result=result, fl_ctx=fl_ctx
+            )
+
+        wait_time = 0
+        while wait_time <= wait_time_after_min_received:
+            assert controller.get_num_standing_tasks() == 1
+            for client in clients:
+                task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+                assert task_name_out == ""
+                assert client_task_id == ""
+            time.sleep(1)
+            wait_time += 1
+
+        assert controller.get_num_standing_tasks() == 0
+        assert task.completion_status == TaskCompletionStatus.OK
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("num_clients", [1, 2, 3, 4])
+    def test_min_resp_is_zero_task_only_exit_when_all_client_task_done(self, method, num_clients):
+        controller, fl_ctx = self.start_controller()
+        clients = [create_client(f"__test_client{i}") for i in range(num_clients)]
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {
+                    "targets": None,
+                    "min_responses": 0,
+                },
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        client_task_ids = []
+        for client in clients:
+            task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+            client_task_ids.append(client_task_id)
+            assert task_name_out == "__test_task"
+            controller._check_tasks()
+            assert controller.get_num_standing_tasks() == 1
+
+        for client, client_task_id in zip(clients, client_task_ids):
+            controller._check_tasks()
+            assert controller.get_num_standing_tasks() == 1
+            result = Shareable()
+            controller.process_submission(
+                client=client, task_name="__test_task", task_id=client_task_id, result=result, fl_ctx=fl_ctx
+            )
+
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 0
+        assert task.completion_status == TaskCompletionStatus.OK
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
diff --git a/test/controller/test_callback.py b/test/controller/test_callback.py
new file mode 100644
index 0000000000..ae4c4fc257
--- /dev/null
+++ b/test/controller/test_callback.py
@@ -0,0 +1,350 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+
+import pytest
+
+from nvflare.apis.controller_spec import ClientTask, Task, TaskCompletionStatus
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+from .controller_test import TestController, create_client, create_task, get_ready, launch_task
+
+
+def _get_task_done_callback_test_cases():
+    task_name = "__test_task"
+
+    def task_done_cb(task: Task, fl_ctx: FLContext):
+        client_names = [x.client.name for x in task.client_tasks]
+        expected_str = "_".join(client_names)
+        task.props[task_name] = expected_str
+
+    input_data = Shareable()
+    test_cases = [
+        [
+            "broadcast",
+            [create_client(f"__test_client{i}") for i in range(10)],
+            task_name,
+            input_data,
+            task_done_cb,
+            "_".join([f"__test_client{i}" for i in range(10)]),
+        ],
+        [
+            "broadcast_and_wait",
+            [create_client(f"__test_client{i}") for i in range(10)],
+            task_name,
+            input_data,
+            task_done_cb,
+            "_".join([f"__test_client{i}" for i in range(10)]),
+        ],
+        ["send", [create_client("__test_client")], task_name, input_data, task_done_cb, "__test_client"],
+        ["send_and_wait", [create_client("__test_client")], task_name, input_data, task_done_cb, "__test_client"],
+        ["relay", [create_client("__test_client")], task_name, input_data, task_done_cb, "__test_client"],
+        ["relay_and_wait", [create_client("__test_client")], task_name, input_data, task_done_cb, "__test_client"],
+    ]
+    return test_cases
+
+
+class TestCallback(TestController):
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    def test_before_task_sent_cb(self, method):
+        def before_task_sent_cb(client_task: ClientTask, fl_ctx: FLContext):
+            client_task.task.data["_test_data"] = client_task.client.name
+
+        client_name = "_test_client"
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name=client_name)
+        task = create_task("__test_task", before_task_sent_cb=before_task_sent_cb)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        get_ready(launch_thread)
+
+        task_name_out, _, data = controller.process_task_request(client, fl_ctx)
+
+        expected = Shareable()
+        expected["_test_data"] = client_name
+        assert data == expected
+        controller.cancel_task(task)
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    def test_result_received_cb(self, method):
+        def result_received_cb(client_task: ClientTask, fl_ctx: FLContext):
+            client_task.result["_test_data"] = client_task.client.name
+
+        client_name = "_test_client"
+        input_data = Shareable()
+        input_data["_test_data"] = "_old_data"
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name=client_name)
+        task = create_task("__test_task", data=input_data, result_received_cb=result_received_cb)
+        kwargs = {"targets": [client]}
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": kwargs,
+            },
+        )
+        get_ready(launch_thread)
+        task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+        controller.process_submission(
+            client=client, task_name="__test_task", task_id=client_task_id, fl_ctx=fl_ctx, result=data
+        )
+
+        expected = Shareable()
+        expected["_test_data"] = client_name
+        assert task.last_client_task_map[client_name].result == expected
+        controller._check_tasks()
+        assert task.completion_status == TaskCompletionStatus.OK
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("task_complete", ["normal", "timeout", "cancel"])
+    @pytest.mark.parametrize("method,clients,task_name,input_data,cb,expected", _get_task_done_callback_test_cases())
+    def test_task_done_cb(self, method, clients, task_name, input_data, cb, expected, task_complete):
+        controller, fl_ctx = self.start_controller()
+
+        timeout = 0 if task_complete != "timeout" else 1
+        task = create_task("__test_task", data=input_data, task_done_cb=cb, timeout=timeout)
+        kwargs = {"targets": clients}
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": kwargs,
+            },
+        )
+        get_ready(launch_thread)
+
+        client_task_ids = len(clients) * [None]
+        for i, client in enumerate(clients):
+            task_name_out, client_task_ids[i], _ = controller.process_task_request(client, fl_ctx)
+
+            if task_name_out == "":
+                client_task_ids[i] = None
+
+        # in here we make up client results:
+        result = Shareable()
+        result["result"] = "result"
+
+        for client, client_task_id in zip(clients, client_task_ids):
+            if client_task_id is not None:
+                if task_complete == "normal":
+                    controller.process_submission(
+                        client=client, task_name="__test_task", task_id=client_task_id, fl_ctx=fl_ctx, result=result
+                    )
+        if task_complete == "timeout":
+            time.sleep(timeout)
+            assert task.completion_status == TaskCompletionStatus.TIMEOUT
+        elif task_complete == "cancel":
+            controller.cancel_task(task)
+            assert task.completion_status == TaskCompletionStatus.CANCELLED
+        controller._check_tasks()
+        assert task.props[task_name] == expected
+        assert controller.get_num_standing_tasks() == 0
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    def test_cancel_task_before_send_cb(self, method):
+        def before_task_sent_cb(client_task: ClientTask, fl_ctx: FLContext):
+            client_task.task.completion_status = TaskCompletionStatus.CANCELLED
+
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+        task = create_task("__test_task", before_task_sent_cb=before_task_sent_cb)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        get_ready(launch_thread)
+
+        task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+        assert task_name_out == ""
+        assert client_task_id == ""
+
+        launch_thread.join()
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    def test_cancel_task_result_received_cb(self, method):
+        def result_received_cb(client_task: ClientTask, fl_ctx: FLContext):
+            client_task.task.completion_status = TaskCompletionStatus.CANCELLED
+
+        controller, fl_ctx = self.start_controller()
+        client1 = create_client(name="__test_client")
+        client2 = create_client(name="__another_client")
+        task = create_task("__test_task", result_received_cb=result_received_cb)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client1, client2]},
+            },
+        )
+        get_ready(launch_thread)
+
+        task_name_out, client_task_id, data = controller.process_task_request(client1, fl_ctx)
+
+        result = Shareable()
+        result["__result"] = "__test_result"
+        controller.process_submission(
+            client=client1, task_name="__test_task", task_id=client_task_id, fl_ctx=fl_ctx, result=result
+        )
+        assert task.last_client_task_map["__test_client"].result == result
+
+        task_name_out, client_task_id, data = controller.process_task_request(client2, fl_ctx)
+        assert task_name_out == ""
+        assert client_task_id == ""
+
+        launch_thread.join()
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    @pytest.mark.parametrize("method2", ["broadcast", "send", "relay"])
+    def test_schedule_task_before_send_cb(self, method, method2):
+        def before_task_sent_cb(client_task: ClientTask, fl_ctx: FLContext):
+            controller = fl_ctx.get_prop(key="controller")
+            new_task = create_task("__new_test_task")
+            inner_launch_thread = threading.Thread(
+                target=launch_task,
+                kwargs={
+                    "controller": controller,
+                    "task": new_task,
+                    "method": method2,
+                    "fl_ctx": fl_ctx,
+                    "kwargs": {"targets": [client_task.client]},
+                },
+            )
+            inner_launch_thread.start()
+            inner_launch_thread.join()
+
+        controller, fl_ctx = self.start_controller()
+        fl_ctx.set_prop("controller", controller)
+        client = create_client(name="__test_client")
+        task = create_task("__test_task", before_task_sent_cb=before_task_sent_cb)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        launch_thread.start()
+
+        task_name_out = ""
+        while task_name_out == "":
+            task_name_out, _, _ = controller.process_task_request(client, fl_ctx)
+            time.sleep(0.1)
+        assert task_name_out == "__test_task"
+        new_task_name_out = ""
+        while new_task_name_out == "":
+            new_task_name_out, _, _ = controller.process_task_request(client, fl_ctx)
+            time.sleep(0.1)
+        assert new_task_name_out == "__new_test_task"
+
+        controller.cancel_task(task)
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    @pytest.mark.parametrize("method2", ["broadcast", "send", "relay"])
+    def test_schedule_task_result_received_cb(self, method, method2):
+        def result_received_cb(client_task: ClientTask, fl_ctx: FLContext):
+            controller = fl_ctx.get_prop(key="controller")
+            new_task = create_task("__new_test_task")
+            inner_launch_thread = threading.Thread(
+                target=launch_task,
+                kwargs={
+                    "controller": controller,
+                    "task": new_task,
+                    "method": method2,
+                    "fl_ctx": fl_ctx,
+                    "kwargs": {"targets": [client_task.client]},
+                },
+            )
+            inner_launch_thread.start()
+            get_ready(inner_launch_thread)
+            inner_launch_thread.join()
+
+        controller, fl_ctx = self.start_controller()
+        fl_ctx.set_prop("controller", controller)
+        client = create_client(name="__test_client")
+        task = create_task("__test_task", result_received_cb=result_received_cb)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        launch_thread.start()
+
+        task_name_out = ""
+        client_task_id = ""
+        data = None
+        while task_name_out == "":
+            task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+            time.sleep(0.1)
+        assert task_name_out == "__test_task"
+
+        controller.process_submission(
+            client=client, task_name="__test_task", task_id=client_task_id, fl_ctx=fl_ctx, result=data
+        )
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 1
+        new_task_name_out = ""
+        while new_task_name_out == "":
+            new_task_name_out, _, _ = controller.process_task_request(client, fl_ctx)
+            time.sleep(0.1)
+        assert new_task_name_out == "__new_test_task"
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
diff --git a/test/controller/test_invalid_input.py b/test/controller/test_invalid_input.py
new file mode 100644
index 0000000000..00dc7698f6
--- /dev/null
+++ b/test/controller/test_invalid_input.py
@@ -0,0 +1,254 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from nvflare.apis.controller_spec import SendOrder
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import Shareable
+
+from .controller_test import TestController, create_client, create_task
+
+
+def _get_common_test_cases():
+    test_cases = [
+        [{"task": list(), "fl_ctx": FLContext()}, TypeError, "task must be an instance of Task."],
+        [
+            {"task": create_task("__test"), "fl_ctx": list()},
+            TypeError,
+            "fl_ctx must be an instance of FLContext.",
+        ],
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "targets": dict()},
+            TypeError,
+            "targets must be a list of Client or string.",
+        ],
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "targets": [1, 2, 3]},
+            TypeError,
+            "targets must be a list of Client or string.",
+        ],
+    ]
+    return test_cases
+
+
+def _get_broadcast_test_cases():
+    test_cases = [
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "min_responses": -1},
+            ValueError,
+            "min_responses must >= 0.",
+        ],
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "min_responses": 1.1},
+            TypeError,
+            "min_responses must be an instance of int.",
+        ],
+    ]
+    return test_cases
+
+
+def _get_send_test_cases():
+    test_cases = [
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "task_assignment_timeout": -1},
+            ValueError,
+            "task_assignment_timeout must >= 0.",
+        ],
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "task_assignment_timeout": 1.1},
+            TypeError,
+            "task_assignment_timeout must be an instance of int.",
+        ],
+        [
+            {
+                "task": create_task("__test"),
+                "fl_ctx": FLContext(),
+                "send_order": SendOrder.SEQUENTIAL,
+                "targets": [],
+            },
+            ValueError,
+            "Targets must be provided for send",
+        ],
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "send_order": "hello"},
+            TypeError,
+            "send_order must be in Enum SendOrder.",
+        ],
+        [
+            {
+                "task": create_task("__test", timeout=2),
+                "fl_ctx": FLContext(),
+                "task_assignment_timeout": 3,
+            },
+            ValueError,
+            "task_assignment_timeout need to be less than or equal to task.timeout.",
+        ],
+    ]
+    return test_cases
+
+
+def _get_relay_test_cases():
+    test_cases = [
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "task_assignment_timeout": -1},
+            ValueError,
+            "task_assignment_timeout must >= 0.",
+        ],
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "task_assignment_timeout": 1.1},
+            TypeError,
+            "task_assignment_timeout must be an instance of int.",
+        ],
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "task_result_timeout": -1},
+            ValueError,
+            "task_result_timeout must >= 0.",
+        ],
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "task_result_timeout": 1.1},
+            TypeError,
+            "task_result_timeout must be an instance of int.",
+        ],
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "send_order": "hello"},
+            TypeError,
+            "send_order must be in Enum SendOrder.",
+        ],
+        [
+            {
+                "task": create_task("__test", timeout=2),
+                "fl_ctx": FLContext(),
+                "task_assignment_timeout": 3,
+            },
+            ValueError,
+            "task_assignment_timeout need to be less than or equal to task.timeout.",
+        ],
+        [
+            {
+                "task": create_task("__test", timeout=2),
+                "fl_ctx": FLContext(),
+                "task_result_timeout": 3,
+            },
+            ValueError,
+            "task_result_timeout need to be less than or equal to task.timeout.",
+        ],
+        [
+            {"task": create_task("__test"), "fl_ctx": FLContext(), "dynamic_targets": False},
+            ValueError,
+            "Need to provide targets when dynamic_targets is set to False.",
+        ],
+    ]
+    return test_cases
+
+
+def _get_process_submission_test_cases():
+    return [
+        [
+            {
+                "client": None,
+                "task_name": "__test_task",
+                "fl_ctx": FLContext(),
+                "task_id": "abc",
+                "result": Shareable(),
+            },
+            TypeError,
+            "client must be an instance of Client.",
+        ],
+        [
+            {
+                "client": create_client("__test"),
+                "task_name": "__test_task",
+                "fl_ctx": None,
+                "task_id": "abc",
+                "result": Shareable(),
+            },
+            TypeError,
+            "fl_ctx must be an instance of FLContext.",
+        ],
+        [
+            {
+                "client": create_client("__test"),
+                "task_name": "__test_task",
+                "fl_ctx": FLContext(),
+                "task_id": "abc",
+                "result": "abc",
+            },
+            TypeError,
+            "result must be an instance of Shareable.",
+        ],
+    ]
+
+
+class TestInvalidInput(TestController):
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    @pytest.mark.parametrize("kwargs,error,msg", _get_common_test_cases())
+    def test_invalid_input(self, method, kwargs, error, msg):
+        controller, fl_ctx = self.start_controller()
+        with pytest.raises(error, match=msg):
+            if method == "broadcast":
+                controller.broadcast(**kwargs)
+            elif method == "broadcast_and_wait":
+                controller.broadcast_and_wait(**kwargs)
+            elif method == "send":
+                controller.send(**kwargs)
+            elif method == "send_and_wait":
+                controller.send_and_wait(**kwargs)
+            elif method == "relay":
+                controller.relay(**kwargs)
+            elif method == "relay_and_wait":
+                controller.relay_and_wait(**kwargs)
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", ["broadcast", "broadcast_and_wait"])
+    @pytest.mark.parametrize("kwargs,error,msg", _get_broadcast_test_cases())
+    def test_broadcast_invalid_input(self, method, kwargs, error, msg):
+        controller, fl_ctx = self.start_controller()
+        with pytest.raises(error, match=msg):
+            if method == "broadcast":
+                controller.broadcast(**kwargs)
+            else:
+                controller.broadcast_and_wait(**kwargs)
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", ["send", "send_and_wait"])
+    @pytest.mark.parametrize("kwargs,error,msg", _get_send_test_cases())
+    def test_send_invalid_input(self, method, kwargs, error, msg):
+        controller, fl_ctx = self.start_controller()
+        with pytest.raises(error, match=msg):
+            if method == "send":
+                controller.send(**kwargs)
+            else:
+                controller.send_and_wait(**kwargs)
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", ["relay", "relay_and_wait"])
+    @pytest.mark.parametrize("kwargs,error,msg", _get_relay_test_cases())
+    def test_relay_invalid_input(self, method, kwargs, error, msg):
+        controller, fl_ctx = self.start_controller()
+        with pytest.raises(error, match=msg):
+            if method == "relay":
+                controller.relay(**kwargs)
+            else:
+                controller.relay_and_wait(**kwargs)
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    @pytest.mark.parametrize("kwargs,error,msg", _get_process_submission_test_cases())
+    def test_process_submission_invalid_input(self, method, kwargs, error, msg):
+        controller, fl_ctx = self.start_controller()
+
+        with pytest.raises(error, match=msg):
+            controller.process_submission(**kwargs)
+        self.stop_controller(controller, fl_ctx)
diff --git a/test/controller/test_relay_behavior.py b/test/controller/test_relay_behavior.py
new file mode 100644
index 0000000000..1bd58701c4
--- /dev/null
+++ b/test/controller/test_relay_behavior.py
@@ -0,0 +1,667 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+from itertools import permutations
+
+import pytest
+
+from nvflare.apis.controller_spec import SendOrder, TaskCompletionStatus
+from nvflare.apis.shareable import ReservedHeaderKey, Shareable
+
+from .controller_test import TestController, create_client, create_task, get_ready, launch_task
+
+
+def _process_task_request_test_cases():
+    """Returns a list of
+    targets, request_client, dynamic_targets, task_assignment_timeout, time_before_first_request,
+    expected_to_get_task, expected_targets
+    """
+    clients = [create_client(f"__test_client{i}") for i in range(3)]
+    client_names = [c.name for c in clients]
+
+    dynamic_targets_cases = [
+        [clients[1:], clients[0], True, 1, 0, False, [clients[1].name, clients[2].name, clients[0].name]],
+        [clients[1:], clients[1], True, 1, 0, True, client_names[1:]],
+        [clients[1:], clients[2], True, 1, 0, False, client_names[1:]],
+        [[clients[0]], clients[1], True, 1, 0, False, [clients[0].name, clients[1].name]],
+        [[clients[0]], clients[1], True, 1, 2, False, [clients[0].name]],
+        [[clients[0], clients[0]], clients[0], True, 1, 0, True, [clients[0].name, clients[0].name]],
+        [None, clients[0], True, 1, 0, True, [clients[0].name]],
+    ]
+
+    static_targets_cases = [
+        [clients[1:], clients[0], False, 1, 0, False, client_names[1:]],
+        [clients[1:], clients[1], False, 1, 0, True, client_names[1:]],
+        [clients[1:], clients[2], False, 1, 0, False, client_names[1:]],
+        [clients[1:], clients[0], False, 1, 2, False, client_names[1:]],
+        [clients[1:], clients[1], False, 1, 2, True, client_names[1:]],
+        [clients[1:], clients[2], False, 1, 2, True, client_names[1:]],
+    ]
+
+    return dynamic_targets_cases + static_targets_cases
+
+
+def _get_sequential_sequence_test_cases():
+    """Returns a list of list of clients"""
+    clients = [create_client(f"__test_client{i}") for i in range(3)]
+    normal_cases = [list(x) for x in permutations(clients)]
+    duplicate_clients = [[clients[0], clients[0]], [clients[0], clients[1], clients[0], clients[2]]]
+    return normal_cases + duplicate_clients
+
+
+def _get_order_with_task_assignment_timeout_test_cases():
+    """Returns a list of
+    send_order, targets, task_assignment_timeout,
+    time_before_first_request, request_orders, expected_clients_to_get_task
+
+    Each item in request_orders is a request_order,
+    In reality, this request order should be random, because each client side does not sync with other clients.
+    """
+    num_clients = 3
+    clients = [create_client(name=f"__test_client{i}") for i in range(num_clients)]
+
+    # these are just helpful orders
+    clients_120 = [clients[1], clients[2], clients[0]]
+    clients_201 = [clients[2], clients[0], clients[1]]
+    clients_210 = [clients[2], clients[1], clients[0]]
+    return [
+        [
+            SendOrder.SEQUENTIAL,
+            clients,
+            2,
+            1,
+            [clients, clients, clients],
+            clients,
+        ],
+        [
+            SendOrder.SEQUENTIAL,
+            clients,
+            2,
+            1,
+            [clients_120, clients_120, clients_120],
+            clients,
+        ],
+        [
+            SendOrder.SEQUENTIAL,
+            clients,
+            2,
+            1,
+            [clients_201, clients_201, clients_201],
+            clients,
+        ],
+        [
+            SendOrder.SEQUENTIAL,
+            clients,
+            2,
+            3,
+            [clients, clients, clients],
+            clients,
+        ],
+        [
+            SendOrder.SEQUENTIAL,
+            clients,
+            2,
+            3,
+            [clients_120, clients_120, clients_120],
+            [clients[1], clients[2], None],
+        ],
+        [
+            SendOrder.SEQUENTIAL,
+            clients,
+            2,
+            3,
+            [clients_201, clients_201, clients_201],
+            clients,
+        ],
+        [
+            SendOrder.SEQUENTIAL,
+            clients,
+            2,
+            3,
+            [clients_210, clients_210, clients_210],
+            [clients[1], clients[2], None],
+        ],
+        [
+            SendOrder.SEQUENTIAL,
+            clients,
+            2,
+            3,
+            [clients_120, clients, clients_120],
+            [clients[1], clients[2], None],
+        ],
+        [
+            SendOrder.SEQUENTIAL,
+            clients,
+            2,
+            5,
+            [clients, clients, clients],
+            clients,
+        ],
+        [
+            SendOrder.SEQUENTIAL,
+            clients,
+            2,
+            5,
+            [clients_120, clients_120, clients_120],
+            [clients[1], clients[2], None],
+        ],
+        [SendOrder.SEQUENTIAL, clients, 2, 5, [clients_201, clients_201, clients_201], [clients[2], None, None]],
+        [SendOrder.SEQUENTIAL, clients, 2, 5, [clients_201, clients, clients_120], [clients[2], None, None]],
+        [
+            SendOrder.SEQUENTIAL,
+            [clients[0], clients[1], clients[2], clients[1], clients[0], clients[0]],
+            2,
+            5,
+            [clients, clients, clients, clients, clients, clients],
+            [clients[0], clients[1], clients[2], clients[1], clients[0], clients[0]],
+        ],
+        [
+            SendOrder.SEQUENTIAL,
+            [clients[0], clients[1], clients[2], clients[1], clients[0], clients[0]],
+            2,
+            5,
+            [clients_201, clients_201, clients_201, clients_201, clients_201, clients_201],
+            [clients[2], clients[1], clients[0], clients[0], None, None],
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            1,
+            [clients, clients, clients],
+            clients,
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            1,
+            [clients_120, clients_120, clients_120],
+            clients_120,
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            1,
+            [clients_201, clients_201, clients_201],
+            clients_201,
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            3,
+            [clients, clients, clients],
+            clients,
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            3,
+            [clients_120, clients_120, clients_120],
+            clients_120,
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            3,
+            [clients_201, clients_201, clients_201],
+            clients_201,
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            3,
+            [clients_210, clients_210, clients_210],
+            clients_210,
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            3,
+            [clients_120, clients, clients_120],
+            [clients[1], clients[0], clients[2]],
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            5,
+            [clients, clients, clients],
+            clients,
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            5,
+            [clients_120, clients_120, clients_120],
+            clients_120,
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            5,
+            [clients_201, clients_201, clients_201],
+            clients_201,
+        ],
+        [
+            SendOrder.ANY,
+            clients,
+            2,
+            5,
+            [clients_201, clients, clients_120],
+            clients_201,
+        ],
+        [
+            SendOrder.ANY,
+            [clients[0], clients[1], clients[2], clients[1], clients[0], clients[0]],
+            2,
+            5,
+            [clients, clients, clients, clients, clients, clients],
+            [clients[0], clients[0], clients[0], clients[1], clients[1], clients[2]],
+        ],
+        [
+            SendOrder.ANY,
+            [clients[0], clients[1], clients[2], clients[1], clients[0], clients[0]],
+            2,
+            5,
+            [clients_201, clients_201, clients_201, clients_201, clients_201, clients_201],
+            [clients[2], clients[0], clients[0], clients[0], clients[1], clients[1]],
+        ],
+        [
+            SendOrder.ANY,
+            [clients[0], clients[1], clients[2], clients[1], clients[0], clients[0]],
+            2,
+            5,
+            [clients_210, clients_210, clients_210, clients_210, clients_201, clients_201, clients],
+            [clients[2], clients[1], clients[1], clients[0], clients[0], clients[0], None],
+        ],
+    ]
+
+
+@pytest.mark.parametrize("method", ["relay", "relay_and_wait"])
+class TestRelayBehavior(TestController):
+    @pytest.mark.parametrize("send_order", [SendOrder.ANY, SendOrder.SEQUENTIAL])
+    def test_only_client_in_target_will_get_task(self, method, send_order):
+        controller, fl_ctx = self.start_controller()
+        clients = [create_client(f"__test_client{i}") for i in range(4)]
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [clients[0]], "send_order": send_order},
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        task_name_out = ""
+        data = None
+        while task_name_out == "":
+            task_name_out, client_task_id, data = controller.process_task_request(clients[0], fl_ctx)
+            time.sleep(0.1)
+        assert task_name_out == "__test_task"
+        assert data == input_data
+        assert task.last_client_task_map[clients[0].name].task_send_count == 1
+        assert controller.get_num_standing_tasks() == 1
+
+        for client in clients[1:]:
+            task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+            assert task_name_out == ""
+            assert client_task_id == ""
+
+        controller.cancel_task(task)
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    def test_task_assignment_timeout_sequential_order_only_client_in_target_will_get_task(self, method):
+        task_assignment_timeout = 3
+        task_result_timeout = 3
+        controller, fl_ctx = self.start_controller()
+        clients = [create_client(f"__test_client{i}") for i in range(4)]
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {
+                    "targets": [clients[0]],
+                    "send_order": SendOrder.SEQUENTIAL,
+                    "task_assignment_timeout": task_assignment_timeout,
+                    "task_result_timeout": task_result_timeout,
+                    "dynamic_targets": False,
+                },
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        time.sleep(task_assignment_timeout + 1)
+
+        for client in clients[1:]:
+            task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+            assert task_name_out == ""
+            assert client_task_id == ""
+
+        controller.cancel_task(task)
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize(
+        "targets,request_client,dynamic_targets,task_assignment_timeout,time_before_first_request,"
+        "expected_to_get_task,expected_targets",
+        _process_task_request_test_cases(),
+    )
+    def test_process_task_request(
+        self,
+        method,
+        targets,
+        request_client,
+        dynamic_targets,
+        task_assignment_timeout,
+        time_before_first_request,
+        expected_to_get_task,
+        expected_targets,
+    ):
+        controller, fl_ctx = self.start_controller()
+        task = create_task("__test_task")
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {
+                    "targets": targets,
+                    "dynamic_targets": dynamic_targets,
+                    "task_assignment_timeout": task_assignment_timeout,
+                },
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+        time.sleep(time_before_first_request)
+
+        task_name, task_id, data = controller.process_task_request(client=request_client, fl_ctx=fl_ctx)
+        client_get_a_task = True if task_name == "__test_task" else False
+
+        assert client_get_a_task == expected_to_get_task
+        assert task.targets == expected_targets
+
+        controller.cancel_task(task)
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("targets", _get_sequential_sequence_test_cases())
+    def test_sequential_sequence(self, method, targets):
+        controller, fl_ctx = self.start_controller()
+        input_data = Shareable()
+        input_data["result"] = "start_"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": targets, "send_order": SendOrder.SEQUENTIAL},
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        expected_client_index = 0
+        while controller.get_num_standing_tasks() != 0:
+            client_tasks_and_results = {}
+
+            for c in targets:
+                task_name, task_id, data = controller.process_task_request(client=c, fl_ctx=fl_ctx)
+                if task_name != "":
+                    client_result = Shareable()
+                    client_result["result"] = f"{c.name}"
+                    if task_id not in client_tasks_and_results:
+                        client_tasks_and_results[task_id] = (c, task_name, client_result)
+                    assert c == targets[expected_client_index]
+
+            for task_id in client_tasks_and_results.keys():
+                c, task_name, client_result = client_tasks_and_results[task_id]
+                task.data["result"] += client_result["result"]
+                controller.process_submission(
+                    client=c, task_name=task_name, task_id=task_id, result=client_result, fl_ctx=fl_ctx
+                )
+                assert task.last_client_task_map[c.name].result == client_result
+            expected_client_index += 1
+
+        launch_thread.join()
+        assert task.data["result"] == "start_" + "".join([c.name for c in targets])
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize(
+        "send_order,targets,task_assignment_timeout,time_before_first_request,request_orders,"
+        "expected_clients_to_get_task",
+        _get_order_with_task_assignment_timeout_test_cases(),
+    )
+    def test_process_request_and_submission_with_task_assignment_timeout(
+        self,
+        method,
+        send_order,
+        targets,
+        request_orders,
+        task_assignment_timeout,
+        time_before_first_request,
+        expected_clients_to_get_task,
+    ):
+        controller, fl_ctx = self.start_controller()
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {
+                    "targets": targets,
+                    "send_order": send_order,
+                    "task_assignment_timeout": task_assignment_timeout,
+                },
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        time.sleep(time_before_first_request)
+
+        for request_order, expected_client_to_get_task in zip(request_orders, expected_clients_to_get_task):
+            task_name_out = ""
+            client_task_id = ""
+            # processing task request
+            for client in request_order:
+                if expected_client_to_get_task and client.name == expected_client_to_get_task.name:
+                    data = None
+                    task_name_out = ""
+                    while task_name_out == "":
+                        task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+                        time.sleep(0.1)
+                    assert task_name_out == "__test_task"
+                    assert data == input_data
+                    assert task.last_client_task_map[client.name].task_send_count == 1
+                else:
+                    _task_name_out, _client_task_id, _ = controller.process_task_request(client, fl_ctx)
+                    assert _task_name_out == ""
+                    assert _client_task_id == ""
+
+            # client side running some logic to generate result
+            if expected_client_to_get_task:
+                controller._check_tasks()
+                assert controller.get_num_standing_tasks() == 1
+                result = Shareable()
+                controller.process_submission(
+                    client=expected_client_to_get_task,
+                    task_name=task_name_out,
+                    task_id=client_task_id,
+                    fl_ctx=fl_ctx,
+                    result=result,
+                )
+
+        launch_thread.join()
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 0
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("send_order", [SendOrder.ANY, SendOrder.SEQUENTIAL])
+    def test_process_submission_after_first_client_task_result_timeout(self, method, send_order):
+        clients = [create_client(name=f"__test_client{i}") for i in range(2)]
+        task_assignment_timeout = 1
+        task_result_timeout = 2
+        controller, fl_ctx = self.start_controller()
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {
+                    "targets": clients,
+                    "send_order": send_order,
+                    "task_assignment_timeout": task_assignment_timeout,
+                    "task_result_timeout": task_result_timeout,
+                },
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        # first client get a task
+        data = None
+        task_name_out = ""
+        old_client_task_id = ""
+        while task_name_out == "":
+            task_name_out, old_client_task_id, data = controller.process_task_request(clients[0], fl_ctx)
+            time.sleep(0.1)
+        assert task_name_out == "__test_task"
+        assert data == input_data
+        assert task.last_client_task_map[clients[0].name].task_send_count == 1
+
+        time.sleep(task_result_timeout + 1)
+
+        # same client ask should get the same task
+        task_name_out, client_task_id, data = controller.process_task_request(clients[0], fl_ctx)
+        assert client_task_id == old_client_task_id
+        assert task.last_client_task_map[clients[0].name].task_send_count == 2
+
+        time.sleep(task_result_timeout + 1)
+
+        # second client ask should get a task since task_result_timeout passed
+        task_name_out, client_task_id_1, data = controller.process_task_request(clients[1], fl_ctx)
+        assert task_name_out == "__test_task"
+        assert data == input_data
+        assert task.last_client_task_map[clients[1].name].task_send_count == 1
+
+        # then we get back first client's result
+        result = Shareable()
+        controller.process_submission(
+            client=clients[0],
+            task_name=task_name_out,
+            task_id=client_task_id,
+            fl_ctx=fl_ctx,
+            result=result,
+        )
+
+        # need to make sure the header is set
+        assert result.get_header(ReservedHeaderKey.REPLY_IS_LATE)
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 1
+
+    @pytest.mark.parametrize("send_order", [SendOrder.ANY, SendOrder.SEQUENTIAL])
+    def test_process_submission_all_client_task_result_timeout(self, method, send_order):
+        clients = [create_client(name=f"__test_client{i}") for i in range(2)]
+        task_assignment_timeout = 1
+        task_result_timeout = 2
+        controller, fl_ctx = self.start_controller()
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {
+                    "targets": clients,
+                    "send_order": send_order,
+                    "task_assignment_timeout": task_assignment_timeout,
+                    "task_result_timeout": task_result_timeout,
+                },
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        # each client get a client task then time out
+        for client in clients:
+            data = None
+            task_name_out = ""
+
+            while task_name_out == "":
+                task_name_out, old_client_task_id, data = controller.process_task_request(client, fl_ctx)
+                time.sleep(0.1)
+            assert task_name_out == "__test_task"
+            assert data == input_data
+            assert task.last_client_task_map[client.name].task_send_count == 1
+
+            time.sleep(task_result_timeout + 1)
+
+        if send_order == SendOrder.SEQUENTIAL:
+            assert task.completion_status == TaskCompletionStatus.TIMEOUT
+            assert controller.get_num_standing_tasks() == 0
+        elif send_order == SendOrder.ANY:
+            assert controller.get_num_standing_tasks() == 1
+            controller.cancel_task(task)
+            assert task.completion_status == TaskCompletionStatus.CANCELLED
+
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
diff --git a/test/controller/test_send_bahavior.py b/test/controller/test_send_bahavior.py
new file mode 100644
index 0000000000..c0bc9cb6b6
--- /dev/null
+++ b/test/controller/test_send_bahavior.py
@@ -0,0 +1,253 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+
+import pytest
+
+from nvflare.apis.controller_spec import SendOrder, TaskCompletionStatus
+from nvflare.apis.shareable import Shareable
+
+from .controller_test import TestController, create_client, create_task, get_ready, launch_task
+
+
+def _assert_other_clients_get_no_task(controller, fl_ctx, client_idx: int, clients):
+    """Assert clients get no task."""
+    assert client_idx < len(clients)
+    for i, client in enumerate(clients):
+        if i == client_idx:
+            continue
+        _task_name_out, _client_task_id, data = controller.process_task_request(client, fl_ctx)
+        assert _task_name_out == ""
+        assert _client_task_id == ""
+
+
+def _get_process_task_request_test_cases():
+    """Returns a lit of
+
+    targets, send_order, request_client_idx
+    """
+    num_clients = 3
+    clients = [create_client(name=f"__test_client{i}") for i in range(num_clients)]
+
+    return [
+        [clients, SendOrder.ANY, 0],
+        [clients, SendOrder.ANY, 1],
+        [clients, SendOrder.ANY, 2],
+        [clients, SendOrder.SEQUENTIAL, 0],
+    ]
+
+
+def _get_process_task_request_with_task_assignment_timeout_test_cases():
+    """Returns a list of
+    targets, send_order, task_assignment_timeout, time_before_first_request, request_order, expected_client_to_get_task
+    """
+    num_clients = 3
+    clients = [create_client(name=f"__test_client{i}") for i in range(num_clients)]
+    clients_120 = [clients[1], clients[2], clients[0]]
+    clients_201 = [clients[2], clients[0], clients[1]]
+    return [
+        [clients, SendOrder.SEQUENTIAL, 2, 1, clients, clients[0].name],
+        [clients, SendOrder.SEQUENTIAL, 2, 1, clients_120, clients[0].name],
+        [clients, SendOrder.SEQUENTIAL, 2, 1, clients_201, clients[0].name],
+        [clients, SendOrder.SEQUENTIAL, 2, 3, clients, clients[0].name],
+        [clients, SendOrder.SEQUENTIAL, 2, 3, clients_120, clients[1].name],
+        [clients, SendOrder.SEQUENTIAL, 2, 3, clients_201, clients[0].name],
+        [clients, SendOrder.SEQUENTIAL, 2, 5, clients, clients[0].name],
+        [clients, SendOrder.SEQUENTIAL, 2, 5, clients_120, clients[1].name],
+        [clients, SendOrder.SEQUENTIAL, 2, 5, clients_201, clients[2].name],
+        [clients, SendOrder.SEQUENTIAL, 2, 3, [clients[2], clients[1], clients[0]], clients[1].name],
+        [clients, SendOrder.ANY, 2, 1, clients, clients[0].name],
+        [clients, SendOrder.ANY, 2, 1, clients_120, clients[1].name],
+        [clients, SendOrder.ANY, 2, 1, clients_201, clients[2].name],
+    ]
+
+
+@pytest.mark.parametrize("method", ["send", "send_and_wait"])
+class TestSendBehavior(TestController):
+    @pytest.mark.parametrize("send_order", [SendOrder.ANY, SendOrder.SEQUENTIAL])
+    def test_process_task_request_client_not_in_target_get_nothing(self, method, send_order):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(f"__test_client")
+        targets = [create_client(f"__target_client")]
+        task = create_task("__test_task")
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": targets, "send_order": send_order},
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        # this client not in target so should get nothing
+        _task_name_out, _client_task_id, data = controller.process_task_request(client, fl_ctx)
+        assert _task_name_out == ""
+        assert _client_task_id == ""
+
+        controller.cancel_task(task)
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 0
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("targets,send_order,client_idx", _get_process_task_request_test_cases())
+    def test_process_task_request_expected_client_get_task_and_unexpected_clients_get_nothing(
+        self, method, targets, send_order, client_idx
+    ):
+        controller, fl_ctx = self.start_controller()
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": targets, "send_order": SendOrder.ANY},
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        # first client
+        task_name_out = ""
+        data = None
+        while task_name_out == "":
+            task_name_out, client_task_id, data = controller.process_task_request(targets[client_idx], fl_ctx)
+            time.sleep(0.1)
+        assert task_name_out == "__test_task"
+        assert data == input_data
+        assert task.last_client_task_map[targets[client_idx].name].task_send_count == 1
+
+        # other clients
+        _assert_other_clients_get_no_task(controller=controller, fl_ctx=fl_ctx, client_idx=client_idx, clients=targets)
+
+        controller.cancel_task(task)
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 0
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize(
+        "targets,send_order,task_assignment_timeout,time_before_first_request,request_order,expected_client_to_get_task",
+        _get_process_task_request_with_task_assignment_timeout_test_cases(),
+    )
+    def test_process_task_request_with_task_assignment_timeout_expected_client_get_task(
+        self,
+        method,
+        targets,
+        send_order,
+        task_assignment_timeout,
+        time_before_first_request,
+        request_order,
+        expected_client_to_get_task,
+    ):
+        controller, fl_ctx = self.start_controller()
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {
+                    "targets": targets,
+                    "send_order": send_order,
+                    "task_assignment_timeout": task_assignment_timeout,
+                },
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        time.sleep(time_before_first_request)
+
+        for client in request_order:
+            data = None
+            if client.name == expected_client_to_get_task:
+                task_name_out = ""
+                while task_name_out == "":
+                    task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+                    time.sleep(0.1)
+                assert task_name_out == "__test_task"
+                assert data == input_data
+                assert task.last_client_task_map[client.name].task_send_count == 1
+            else:
+                task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+                assert task_name_out == ""
+                assert client_task_id == ""
+
+        controller.cancel_task(task)
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("num_of_clients", [1, 2, 3])
+    def test_send_only_one_task_and_exit_when_client_task_done(self, method, num_of_clients):
+        controller, fl_ctx = self.start_controller()
+        clients = [create_client(f"__test_client{i}") for i in range(num_of_clients)]
+        input_data = Shareable()
+        input_data["hello"] = "world"
+        task = create_task("__test_task", data=input_data)
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": clients, "send_order": SendOrder.SEQUENTIAL},
+            },
+        )
+        get_ready(launch_thread)
+        assert controller.get_num_standing_tasks() == 1
+
+        # first client
+        task_name_out = ""
+        client_task_id = ""
+        data = None
+        while task_name_out == "":
+            task_name_out, client_task_id, data = controller.process_task_request(clients[0], fl_ctx)
+            time.sleep(0.1)
+        assert task_name_out == "__test_task"
+        assert data == input_data
+        assert task.last_client_task_map[clients[0].name].task_send_count == 1
+
+        # once a client gets a task, other clients should not get task
+        _assert_other_clients_get_no_task(controller=controller, fl_ctx=fl_ctx, client_idx=0, clients=clients)
+
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 1
+
+        controller.process_submission(
+            client=clients[0], task_name="__test_task", task_id=client_task_id, fl_ctx=fl_ctx, result=data
+        )
+
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 0
+        assert task.completion_status == TaskCompletionStatus.OK
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
diff --git a/test/controller/test_task.py b/test/controller/test_task.py
new file mode 100644
index 0000000000..91fed00476
--- /dev/null
+++ b/test/controller/test_task.py
@@ -0,0 +1,75 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from .controller_test import create_task
+
+
+def _get_create_task_cases():
+    test_cases = [
+        [
+            {"timeout": -1},
+            ValueError,
+            "timeout must >= 0.",
+        ],
+        [
+            {"timeout": 1.1},
+            TypeError,
+            "timeout must be an instance of int.",
+        ],
+        [
+            {"before_task_sent_cb": list()},
+            TypeError,
+            "before_task_sent must be a callable function.",
+        ],
+        [
+            {"result_received_cb": list()},
+            TypeError,
+            "result_received must be a callable function.",
+        ],
+        [
+            {"task_done_cb": list()},
+            TypeError,
+            "task_done must be a callable function.",
+        ],
+    ]
+    return test_cases
+
+
+class TestTask:
+    @pytest.mark.parametrize("kwargs,error,msg", _get_create_task_cases())
+    def test_create_task_with_invalid_input(self, kwargs, error, msg):
+        with pytest.raises(error, match=msg):
+            _ = create_task(name="__test_task", **kwargs)
+
+    def test_set_task_prop(self):
+        task = create_task(name="__test_task")
+        task.set_prop("hello", "world")
+        assert task.props["hello"] == "world"
+
+    def test_get_task_prop(self):
+        task = create_task(name="__test_task")
+        task.props["hello"] = "world"
+        assert task.get_prop("hello") == "world"
+
+    def test_set_task_prop_invalid_key(self):
+        task = create_task(name="__test_task")
+        with pytest.raises(ValueError, match="Keys start with __ is reserved. Please use other key."):
+            task.set_prop("__test", "world")
+
+    def test_get_task_prop_invalid_key(self):
+        task = create_task(name="__test_task")
+        with pytest.raises(ValueError, match="Keys start with __ is reserved. Please use other key."):
+            task.get_prop("__test")
diff --git a/test/controller/test_task_mgmt.py b/test/controller/test_task_mgmt.py
new file mode 100644
index 0000000000..85b0198d9e
--- /dev/null
+++ b/test/controller/test_task_mgmt.py
@@ -0,0 +1,182 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+import time
+
+import pytest
+
+from nvflare.apis.controller_spec import TaskCompletionStatus
+from nvflare.apis.shareable import Shareable
+
+from .controller_test import TestController, create_client, create_task, get_ready, launch_task
+
+
+class TestTaskManagement(TestController):
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    @pytest.mark.parametrize("num_of_tasks", [2, 3, 4])
+    def test_add_task(self, method, num_of_tasks):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+
+        all_threads = []
+        all_tasks = []
+        for i in range(num_of_tasks):
+            task = create_task(name="__test_task")
+            launch_thread = threading.Thread(
+                target=launch_task,
+                kwargs={
+                    "controller": controller,
+                    "task": task,
+                    "method": method,
+                    "fl_ctx": fl_ctx,
+                    "kwargs": {"targets": [client]},
+                },
+            )
+            get_ready(launch_thread)
+            all_threads.append(launch_thread)
+            all_tasks.append(task)
+        assert controller.get_num_standing_tasks() == num_of_tasks
+        controller.cancel_all_tasks()
+        for task in all_tasks:
+            assert task.completion_status == TaskCompletionStatus.CANCELLED
+        for thread in all_threads:
+            thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method1", TestController.ALL_APIS)
+    @pytest.mark.parametrize("method2", TestController.ALL_APIS)
+    def test_reuse_same_task(self, method1, method2):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+        task = create_task(name="__test_task")
+        targets = [client]
+
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method1,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": targets},
+            },
+        )
+        get_ready(launch_thread)
+
+        with pytest.raises(ValueError, match="Task was already used. Please create a new task object."):
+            launch_task(controller=controller, method=method2, task=task, fl_ctx=fl_ctx, kwargs={"targets": targets})
+
+        controller.cancel_task(task)
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    @pytest.mark.parametrize("num_of_start_tasks", [2, 3, 4])
+    @pytest.mark.parametrize("num_of_cancel_tasks", [1, 2])
+    def test_check_task_remove_cancelled_tasks(self, method, num_of_start_tasks, num_of_cancel_tasks):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+
+        all_threads = []
+        all_tasks = []
+        for i in range(num_of_start_tasks):
+            task = create_task(name=f"__test_task{i}")
+            launch_thread = threading.Thread(
+                target=launch_task,
+                kwargs={
+                    "controller": controller,
+                    "task": task,
+                    "method": method,
+                    "fl_ctx": fl_ctx,
+                    "kwargs": {"targets": [client]},
+                },
+            )
+            get_ready(launch_thread)
+            all_threads.append(launch_thread)
+            all_tasks.append(task)
+
+        for i in range(num_of_cancel_tasks):
+            controller.cancel_task(task=all_tasks[i], fl_ctx=fl_ctx)
+            assert all_tasks[i].completion_status == TaskCompletionStatus.CANCELLED
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == (num_of_start_tasks - num_of_cancel_tasks)
+        controller.cancel_all_tasks()
+        for thread in all_threads:
+            thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    @pytest.mark.parametrize("num_client_requests", [1, 2, 3, 4])
+    def test_client_request_after_cancel_task(self, method, num_client_requests):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+        task = create_task("__test_task")
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        get_ready(launch_thread)
+        controller.cancel_task(task)
+        for i in range(num_client_requests):
+            _, task_id, data = controller.process_task_request(client, fl_ctx)
+            # check if task_id is empty means this task is not assigned
+            assert task_id == ""
+            assert data is None
+        controller._check_tasks()
+        assert controller.get_num_standing_tasks() == 0
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
+
+    @pytest.mark.parametrize("method", TestController.ALL_APIS)
+    def test_client_submit_result_after_cancel_task(self, method):
+        controller, fl_ctx = self.start_controller()
+        client = create_client(name="__test_client")
+        task = create_task("__test_task")
+        launch_thread = threading.Thread(
+            target=launch_task,
+            kwargs={
+                "controller": controller,
+                "task": task,
+                "method": method,
+                "fl_ctx": fl_ctx,
+                "kwargs": {"targets": [client]},
+            },
+        )
+        get_ready(launch_thread)
+        task_name_out, client_task_id, data = controller.process_task_request(client, fl_ctx)
+        controller.cancel_task(task)
+        assert task.completion_status == TaskCompletionStatus.CANCELLED
+        time.sleep(1)
+
+        # in here we make up client results:
+        result = Shareable()
+        result["result"] = "result"
+
+        with pytest.raises(RuntimeError, match=f"Unknown task: __test_task from client __test_client."):
+            controller.process_submission(
+                client=client, task_name="__test_task", task_id=client_task_id, fl_ctx=fl_ctx, result=result
+            )
+
+        assert task.last_client_task_map["__test_client"].result is None
+        launch_thread.join()
+        self.stop_controller(controller, fl_ctx)
diff --git a/test/test_aggregator.py b/test/test_aggregator.py
new file mode 100644
index 0000000000..5518de5375
--- /dev/null
+++ b/test/test_aggregator.py
@@ -0,0 +1,158 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+
+import numpy as np
+import pytest
+
+from nvflare.apis.fl_constant import ReservedKey
+from nvflare.apis.fl_context import FLContext, FLContextManager
+
+from nvflare.apis.shareable import Shareable, ReservedHeaderKey
+from nvflare.app_common.aggregators.accumulate_model_aggregator import AccumulateWeightedAggregator
+from nvflare.app_common.aggregators.intime_accumulate_model_aggregator import InTimeAccumulateWeightedAggregator
+
+# from nvflare.app_common.app_constant import AppConstants, AppShareableKey, AppShareableValue, ShareableContentType
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.apis.dxo import DXO, DataKind, MetaKey, from_shareable
+
+
+class TestAggregator:
+    @pytest.mark.parametrize("aggregator", [AccumulateWeightedAggregator, InTimeAccumulateWeightedAggregator])
+    @pytest.mark.parametrize(
+        "received,expected",
+        [
+            [
+                {"client1": {"weight": 0.5, "iter_number": 1, "aggr_data": {"var1": np.array([2.0, 3.0, 1.1, 0.1])}}},
+                {"var1": np.array([1.0, 1.5, 0.55, 0.05])},
+            ],
+            [
+                {"client1": {"weight": 1.0, "iter_number": 1, "aggr_data": {"var1": np.array([2.0, 3.0, 1.1, 0.1])}}},
+                {"var1": np.array([2.0, 3.0, 1.1, 0.1])},
+            ],
+            [
+                {
+                    "client1": {"weight": 0.5, "iter_number": 3, "aggr_data": {"var1": np.array([2.0, 3.0, 1.1, 0.1])}},
+                    "client2": {"weight": 1.0, "iter_number": 1, "aggr_data": {"var1": np.array([1.0, 1.0, 2.1, 0.5])}},
+                },
+                {"var1": np.array([1.0, 1.375, 0.9375, 0.1625])},
+            ],
+            [
+                {
+                    "client1": {"weight": 1.0, "iter_number": 5, "aggr_data": {"var1": np.array([2.0, 3.0, 1.1, 0.1])}},
+                    "client2": {"weight": 0.5, "iter_number": 4, "aggr_data": {"var1": np.array([1.0, 1.0, 2.1, 0.5])}},
+                },
+                {"var1": np.array([4.0 / 3, 17.0 / 9, 9.7 / 9, 1.0 / 6])},
+            ],
+        ],
+    )
+    def test_accum_aggregator(self, aggregator, received, expected):
+        aggregation_weights = {k: v["weight"] for k, v in received.items()}
+        agg = aggregator(aggregation_weights=aggregation_weights)
+        fl_ctx_mgr = FLContextManager(engine=None, identity_name="", run_num=1, public_stickers={}, private_stickers={})
+        for k, v in received.items():
+            fl_ctx_mgr.identity_name = k
+            dxo = DXO(
+                DataKind.WEIGHT_DIFF,
+                data=v["aggr_data"],
+                meta={
+                    MetaKey.NUM_STEPS_CURRENT_ROUND: v["iter_number"],
+                },
+            )
+
+            fl_ctx = FLContext()
+            fl_ctx.set_prop(AppConstants.CURRENT_ROUND, 0)
+            s = Shareable()
+            s.set_peer_props({ReservedKey.IDENTITY_NAME: k})
+            s.set_header(AppConstants.CONTRIBUTION_ROUND, 0)
+            agg.accept(dxo.update_shareable(s), fl_ctx)
+
+        result = agg.aggregate(fl_ctx)
+        np.testing.assert_allclose(result["DXO"]["data"]["var1"], expected["var1"])
+
+    @pytest.mark.parametrize("aggregator", [AccumulateWeightedAggregator, InTimeAccumulateWeightedAggregator])
+    @pytest.mark.parametrize("shape", [(4), (6, 6)])
+    @pytest.mark.parametrize("n_clients", [10, 50, 100])
+    def test_accum_aggregator_random(self, aggregator, shape, n_clients):
+        aggregation_weights = {f"client_{i}": random.random() for i in range(n_clients)}
+        agg = aggregator(aggregation_weights=aggregation_weights)
+        weighted_sum = np.zeros(shape)
+        sum_of_weights = 0
+        fl_ctx_mgr = FLContextManager(engine=None, identity_name="", run_num=1, public_stickers={}, private_stickers={})
+        for client_name in aggregation_weights:
+            iter_number = random.randint(1, 50)
+            fl_ctx_mgr.identity_name = client_name
+            weights = np.random.random(shape)
+            fl_ctx = FLContext()
+            fl_ctx.set_prop(AppConstants.CURRENT_ROUND, 0)
+            s = Shareable()
+            s.set_peer_props({ReservedKey.IDENTITY_NAME: client_name})
+            s.set_header(AppConstants.CONTRIBUTION_ROUND, 0)
+            dxo = DXO(
+                DataKind.WEIGHT_DIFF,
+                data={"var1": weights},
+                meta={
+                    MetaKey.NUM_STEPS_CURRENT_ROUND: iter_number,
+                },
+            )
+            weighted_sum = weighted_sum + weights * iter_number * aggregation_weights[client_name]
+            sum_of_weights = sum_of_weights + iter_number
+            agg.accept(dxo.update_shareable(s), fl_ctx)
+
+        result = agg.aggregate(fl_ctx)
+        result_dxo = from_shareable(result)
+        np.testing.assert_allclose(result_dxo.data["var1"], weighted_sum / sum_of_weights)
+
+    @pytest.mark.parametrize("aggregator", [AccumulateWeightedAggregator, InTimeAccumulateWeightedAggregator])
+    def test_accum_aggregator_accept(self, aggregator):
+        aggregation_weights = {f"client_{i}": random.random() for i in range(2)}
+        agg = aggregator(aggregation_weights=aggregation_weights)
+        client_name = "client_0"
+        iter_number = 1
+        fl_ctx_mgr = FLContextManager(engine=None, identity_name="", run_num=1, public_stickers={}, private_stickers={})
+        fl_ctx_mgr.identity_name = client_name
+        weights = np.random.random((4))
+
+        fl_ctx = FLContext()
+        s = Shareable()
+        s.set_peer_props({ReservedKey.IDENTITY_NAME: client_name})
+        s.set_header(AppConstants.CONTRIBUTION_ROUND, 1)
+        fl_ctx.set_prop(AppConstants.CURRENT_ROUND, 1)
+        dxo = DXO(
+            DataKind.WEIGHT_DIFF,
+            data={"var1": weights},
+            meta={
+                MetaKey.NUM_STEPS_CURRENT_ROUND: iter_number,
+            },
+        )
+        assert (True) == agg.accept(dxo.update_shareable(s), fl_ctx)
+
+        client_name = "client_0"
+        iter_number = 1
+        fl_ctx_mgr.identity_name = client_name
+        weights = np.random.random((4))
+        dxo = DXO(
+            DataKind.WEIGHT_DIFF,
+            data={"var1": weights},
+            meta={
+                MetaKey.NUM_STEPS_CURRENT_ROUND: iter_number,
+            },
+        )
+        fl_ctx = FLContext()
+        s = Shareable()
+        s.set_peer_props({ReservedKey.IDENTITY_NAME: client_name})
+        s.set_header(AppConstants.CONTRIBUTION_ROUND, 1)
+        fl_ctx.set_prop(AppConstants.CURRENT_ROUND, 2)
+        assert (False) == agg.accept(dxo.update_shareable(s), fl_ctx)
diff --git a/test/test_exclude_vars.py b/test/test_exclude_vars.py
new file mode 100644
index 0000000000..55bc8851b1
--- /dev/null
+++ b/test/test_exclude_vars.py
@@ -0,0 +1,32 @@
+import pytest
+
+from nvflare.apis.dxo import DXO, DataKind, from_shareable
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.filters import ExcludeVars
+
+TEST_CASES = [
+    [{"a": 1.0, "b": 2.0}, "a", {"b": 2.0}],
+    [{"a": 1.0, "b": 2.0, "c": 3.0}, ["a", "b"], {"c": 3.0}],
+    [{"a": 1.0, "b": 2.0, "c": 3.0, "d": 4.0}, ["a", "d"], {"b": 2.0, "c": 3.0}],
+    [
+        {"conv/a": 1.0, "conv/b": 2.0, "drop/c": 3.0, "conv/d": 4.0},
+        ["conv/*"],
+        {"conv/a": 1.0, "conv/b": 2.0, "drop/c": 3.0, "conv/d": 4.0},
+    ],
+    [{"conv/a": 1.0, "conv/b": 2.0, "drop/c": 3.0, "conv/d": 4.0}, "conv/*", {"drop/c": 3.0}],
+]
+
+
+class TestExcludeVars:
+    @pytest.mark.parametrize("input_data,exclude_vars,expected_data", TEST_CASES)
+    def test_exclude(self, input_data, exclude_vars, expected_data):
+        dxo = DXO(
+            data_kind=DataKind.WEIGHTS,
+            data=input_data,
+        )
+        data = dxo.to_shareable()
+        fl_ctx = FLContext()
+        f = ExcludeVars(exclude_vars=exclude_vars)
+        new_data = f.process(data, fl_ctx)
+        new_dxo = from_shareable(new_data)
+        assert new_dxo.data == expected_data
diff --git a/test/test_fl_context.py b/test/test_fl_context.py
new file mode 100644
index 0000000000..49dbe7d974
--- /dev/null
+++ b/test/test_fl_context.py
@@ -0,0 +1,79 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.apis.fl_context import FLContext
+
+
+class TestFLContext:
+    def test_add_item(self):
+
+        expected = {"x": 1, "y": 2}
+
+        fl_ctx = FLContext()
+
+        fl_ctx.set_prop("x", 1, private=False)
+        fl_ctx.set_prop("y", 2, private=False)
+
+        assert fl_ctx.get_prop("y") == 2
+
+        assert fl_ctx.get_all_public_props() == expected
+
+    def test_add_with_private_item(self):
+
+        expected = {"x": 1, "y": 2}
+
+        fl_ctx = FLContext()
+
+        fl_ctx.set_prop("x", 1, private=False)
+        fl_ctx.set_prop("y", 2, private=False)
+        fl_ctx.set_prop("z", 3, private=True)
+
+        assert fl_ctx.get_prop("y") == 2
+
+        assert fl_ctx.get_all_public_props() == expected
+
+    def test_set_items(self):
+
+        expected = {"_public_x": 1, "_public_y": 2}
+
+        fl_ctx = FLContext()
+
+        fl_ctx.set_prop("z", 3, private=False)
+
+        # Overwrite the existing public_props.
+        fl_ctx.set_public_props(expected)
+
+        assert fl_ctx.get_all_public_props() == expected
+
+    def test_not_allow_duplicate_key(self):
+        fl_ctx = FLContext()
+
+        fl_ctx.set_prop("x", 1, private=False)
+        fl_ctx.set_prop("y", 2, private=False)
+        fl_ctx.set_prop("z", 3, private=True)
+
+        assert fl_ctx.set_prop("y", 20, private=False)
+        assert fl_ctx.get_prop("y") == 20
+        assert not fl_ctx.set_prop("y", 4, private=True)
+
+    def test_remove_prop(self):
+        fl_ctx = FLContext()
+
+        assert fl_ctx.set_prop("x", 1, private=False)
+        assert fl_ctx.set_prop("y", 2, private=False)
+
+        fl_ctx.remove_prop("y")
+        assert fl_ctx.set_prop("y", 20, private=True)
+
+        assert fl_ctx.get_prop("y") == 20
diff --git a/test/test_format_check.py b/test/test_format_check.py
new file mode 100644
index 0000000000..13e312f425
--- /dev/null
+++ b/test/test_format_check.py
@@ -0,0 +1,57 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from nvflare.apis.utils.format_check import name_check
+
+
+class TestNameCheck:
+    @pytest.mark.parametrize("name, err_value", [["bad***", True], ["bad?!", True], ["bad{}", True], ["good", False]])
+    def test_org(self, name, err_value):
+
+        err, reason = name_check(name, "org")
+        assert err == err_value
+
+    @pytest.mark.parametrize(
+        "name, err_value",
+        [
+            ["localhost", False],
+            ["mylocalmachine", False],
+            ["bad_name", True],
+            [" badname", True],
+            ["bad_name.com", True],
+            ["good-name.com", False],
+        ],
+    )
+    def test_server(self, name, err_value):
+
+        err, reason = name_check(name, "server")
+        assert err == err_value
+
+    @pytest.mark.parametrize("name, err_value", [["*.-", True], ["good-name", False], ["good_name", False]])
+    def test_client(self, name, err_value):
+
+        err, reason = name_check(name, "client")
+        assert err == err_value
+
+    @pytest.mark.parametrize(
+        "name, err_value", [["bad_email*", True], ["bad_email", True], ["bad_email@", True], ["bad_email@123", True]]
+    )
+    def test_admin(self, name, err_value):
+
+        err, reason = name_check(name, "admin")
+        assert err == err_value
+        err, reason = name_check(name, "email")
+        assert err == err_value
diff --git a/test/test_jsonscaner.py b/test/test_jsonscaner.py
new file mode 100644
index 0000000000..5b1122e15d
--- /dev/null
+++ b/test/test_jsonscaner.py
@@ -0,0 +1,252 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import unittest
+
+from nvflare.fuel.utils.json_scanner import JsonObjectProcessor, JsonScanner, Node
+
+
+class _TestJsonProcessor(JsonObjectProcessor):
+    def __init__(self):
+        JsonObjectProcessor.__init__(self)
+
+    def process_element(self, node: Node):
+        pats = [
+            r".\.pre_transforms\.#[0-9]+$",
+            r"^train\.model\.name$",
+            r".\.search\.#[0-9]+$",
+            r".\.pre_transforms\.#[0-9]+\.args$",
+        ]
+        path = node.path()
+        print("ENTER Level: {}; Key: {}; Pos: {}; Path: {}".format(node.level, node.key, node.position, path))
+        for p in pats:
+            x = re.search(p, path)
+            if x:
+                print("\t {} matches {}".format(path, p))
+
+        node.exit_cb = self.post_process_element
+
+    def post_process_element(self, node: Node):
+        path = node.path()
+        print("EXIT Level: {}; Key: {}; Pos: {}; Path: {}".format(node.level, node.key, node.position, path))
+
+
+def _test():
+    import json
+
+    test_json = """
+    {
+     "learning_rate": 1e-4,
+     "lr_search" : [1e-4, 2e-3],
+     "train": {
+      "model": {
+        "name": "SegAhnet",
+        "args": {
+          "num_classes": 2,
+          "if_use_psp": false,
+          "pretrain_weight_name": "{PRETRAIN_WEIGHTS_FILE}",
+          "plane": "z",
+          "final_activation": "softmax",
+          "n_spatial_dim": 3
+        },
+        "search": [
+          {
+            "type": "float",
+            "args": ["num_classes"],
+            "targets": [1,3],
+            "domain": "net"
+          },
+          {
+            "type": "float",
+            "args": ["n_spatial_dim"],
+            "targets": [2,5],
+            "domain": "net"
+          },
+          {
+            "type": "enum",
+            "args": ["n_spatial_dim", "num_classes"],
+            "targets": [[2,3],[3,4],[5,1]],
+            "domain": "net"
+          },
+          {
+            "type": "enum",
+            "args": ["n_spatial_dim"],
+            "targets": [[2],[3],[6],[12]],
+            "domain": "net"
+          }
+        ]
+      },
+      "pre_transforms": [
+      {
+        "name": "LoadNifti",
+        "args": {
+          "fields": [
+            "image",
+            "label"
+          ]
+        }
+      },
+      {
+        "name": "ConvertToChannelsFirst",
+        "args": {
+          "fields": [
+            "image",
+            "label"
+          ]
+        }
+      },
+      {
+        "name": "ScaleIntensityRange",
+        "args": {
+          "fields": "image",
+          "a_min": -57,
+          "a_max": 164,
+          "b_min": 0.0,
+          "b_max": 1.0,
+          "clip": true
+        }
+      },
+      {
+        "name": "FastCropByPosNegRatio",
+        "args": {
+          "size": [
+            96,
+            96,
+            96
+          ],
+          "fields": "image",
+          "label_field": "label",
+          "pos": 1,
+          "neg": 1,
+          "batch_size": 3
+        },
+        "search": [
+          {
+            "domain": "transform",
+            "type": "enum",
+            "args": ["size"],
+            "targets": [[[32, 32, 32]], [[64, 64, 64]], [[128, 128, 128]]]
+          },
+          {
+            "domain": "transform",
+            "type": "enum",
+            "args": ["batch_size"],
+            "targets": [[3], [4], [8], [10]]
+          }
+        ]
+      },
+      {
+        "name": "RandomAxisFlip",
+        "args": {
+          "fields": [
+            "image",
+            "label"
+          ],
+          "probability": 0.0
+        },
+        "search": [
+          {
+            "domain": "transform",
+            "type": "float",
+            "args": ["probability#p"],
+            "targets": [0.0, 1.0]
+          },
+          {
+            "domain": "transform",
+            "args": "DISABLED"
+          }
+        ]
+      },
+      {
+        "name": "RandomRotate3D",
+        "args": {
+          "fields": [
+            "image",
+            "label"
+          ],
+          "probability": 0.0
+        }
+      },
+      {
+        "name": "ScaleIntensityOscillation",
+        "args": {
+          "fields": "image",
+          "magnitude": 0.10
+        }
+      },
+      {
+        "name": "LoadNifti",
+        "args": {
+          "fields": [
+            "image",
+            "label"
+          ]
+        }
+      },
+      {
+        "name": "LoadNifti",
+        "args": {
+          "fields": [
+            "image",
+            "label"
+          ]
+        }
+      },
+      {
+        "name": "LoadNifti",
+        "args": {
+          "fields": [
+            "image",
+            "label"
+          ]
+        }
+      },
+      {
+        "name": "LoadNifti",
+        "args": {
+          "fields": [
+            "image",
+            "label"
+          ]
+        }
+      },
+      {
+        "name": "RandomAxisFlip",
+        "args": {
+          "fields": [
+            "image",
+            "label"
+          ],
+          "probability": 0.0
+        },
+        "search": [
+          {
+            "domain": "transform",
+            "type": "float",
+            "args": ["probability#p"],
+            "targets": [0.0, 1.0]
+          },
+          {
+            "domain": "transform",
+            "args": "DISABLED"
+          }
+        ]
+      }
+    ]
+    } }
+    """
+    train_config = json.loads(test_json)
+    scanner = JsonScanner(train_config, "test")
+    scanner.scan(_TestJsonProcessor())
diff --git a/test/test_messageproto.py b/test/test_messageproto.py
new file mode 100644
index 0000000000..acfd372e7d
--- /dev/null
+++ b/test/test_messageproto.py
@@ -0,0 +1,25 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nvflare.private.admin_defs import Message
+from nvflare.private.fed.utils.messageproto import message_to_proto, proto_to_message
+
+
+class TestMessageProto:
+    def test_message_proto_convert(self):
+        message = Message(topic="topic", body="{'id': 100}")
+        message.set_header("Content-Type", "application/json")
+        message_proto = message_to_proto(message)
+        new_message = proto_to_message(message_proto)
+        assert new_message.__dict__ == message.__dict__
diff --git a/test/test_model_selection.py b/test/test_model_selection.py
new file mode 100644
index 0000000000..67e8f7f30d
--- /dev/null
+++ b/test/test_model_selection.py
@@ -0,0 +1,91 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext, FLContextManager
+from nvflare.app_common.app_constant import AppConstants
+from nvflare.app_common.app_event_type import AppEventType
+from nvflare.app_common.handlers.intime_model_selection_handler import IntimeModelSelectionHandler
+from nvflare.apis.dxo import DXO, DataKind, MetaKey
+
+
+class MockSimpleEngine:
+    def __init__(self, run_num=0):
+        self.fl_ctx_mgr = FLContextManager(
+            engine=self,
+            identity_name="__mock_simple_engine",
+            run_num=run_num,
+            public_stickers={},
+            private_stickers={},
+        )
+        self.last_event = None
+
+    def new_context(self):
+        return self.fl_ctx_mgr.new_context()
+
+    def fire_event(self, event_type: str, fl_ctx: FLContext):
+        self.last_event = event_type
+        return True
+
+
+class TestModelSelectionHandler:
+    @pytest.mark.parametrize("mshandler", [IntimeModelSelectionHandler])
+    @pytest.mark.parametrize(
+        "initial,received,expected",
+        [
+            [
+                1,
+                {
+                    "client1": {"weight": 0.5, "iter_number": 1, "metric": 10},
+                },
+                True,
+            ],
+            [
+                1,
+                {
+                    "client1": {"weight": 0.5, "iter_number": 1, "metric": 1},
+                    "client2": {"weight": 0.5, "iter_number": 1, "metric": 0.2},
+                },
+                False,
+            ],
+        ],
+    )
+    def test_model_section(self, mshandler, initial, received, expected):
+        aggregation_weights = {k: v["weight"] for k, v in received.items()}
+        handler = mshandler(aggregation_weights=aggregation_weights)
+        handler.best_val_metric = initial
+        engine = MockSimpleEngine()
+        for k, v in received.items():
+            peer_ctx = FLContext()
+            peer_ctx.set_prop(FLContextKey.CLIENT_NAME, k, private=False)
+            dxo = DXO(
+                DataKind.WEIGHT_DIFF,
+                data=dict(),
+                meta={
+                    MetaKey.INITIAL_METRICS: v["metric"],
+                    MetaKey.NUM_STEPS_CURRENT_ROUND: v["iter_number"],
+                    AppConstants.CURRENT_ROUND: 10,
+                },
+            )
+
+            peer_ctx.set_prop(FLContextKey.SHAREABLE, dxo.to_shareable(), private=False)
+            fl_ctx = engine.fl_ctx_mgr.new_context()
+            fl_ctx.set_prop(FLContextKey.PEER_CONTEXT, peer_ctx)
+
+            handler.handle_event(EventType.BEFORE_PROCESS_SUBMISSION, fl_ctx)
+        handler.handle_event(AppEventType.BEFORE_AGGREGATION, fl_ctx)
+        assert (engine.last_event == AppEventType.GLOBAL_BEST_MODEL_AVAILABLE) == expected
diff --git a/test/test_security_utils.py b/test/test_security_utils.py
new file mode 100644
index 0000000000..3bc87a37c7
--- /dev/null
+++ b/test/test_security_utils.py
@@ -0,0 +1,49 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+import pytest
+
+from nvflare.fuel.hci.security import hash_password, make_session_token, verify_password
+
+
+class TestSecurityUtils:
+    @pytest.mark.parametrize("password_to_hash", ["abcde", "xyz"])
+    def test_hash_password(self, password_to_hash):
+        hashed_password = hash_password(password_to_hash)
+        assert verify_password(hashed_password, password_to_hash)
+
+    @pytest.mark.parametrize(
+        "password_hash, password_to_verify, expected_result",
+        [
+            [
+                "b1a020416bc1479da3f937af46f90a09a4c09cd6271609105f80e7c9e7fd461ffe463784834ea63c7525f85b80435bbc0dfba614570f23aaccbd115bbef81a57b2e73a39563f0d1b75132c8b9e1b53dc94a1525be01d0e6862e577360e820592",
+                "abcde",
+                False,
+            ],
+            [
+                "b1a020416bc1479da3f937af46f90a09a4c09cd6271609105f80e7c9e7fd461ffe463784834ea63c7525f85b80435bbc0dfba614570f23aaccbd115bbef81a57b2e73a39563f0d1b75132c8b9e1b53dc94a1525be01d0e6862e577360e820592",
+                "xyz",
+                True,
+            ],
+        ],
+    )
+    def test_verify_password(self, password_hash, password_to_verify, expected_result):
+        result = verify_password(password_hash, password_to_verify)
+        assert result == expected_result
+
+    def test_make_session_token(self):
+        uuid.UUID(make_session_token())
+        assert True