From 1f4551dd3f9e007f52b9b5d21b80ec93f0a3bd6e Mon Sep 17 00:00:00 2001 From: Matthew Welborn Date: Tue, 13 Aug 2019 15:23:45 -0400 Subject: [PATCH 1/3] Updates to snowflake and quickstart notebook as part of issue #289 --- docs/qcfractal/source/quickstart.ipynb | 28 +++++++++++++++++++------- qcfractal/snowflake.py | 4 +++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/qcfractal/source/quickstart.ipynb b/docs/qcfractal/source/quickstart.ipynb index f0959164f..1ffaa6d72 100644 --- a/docs/qcfractal/source/quickstart.ipynb +++ b/docs/qcfractal/source/quickstart.ipynb @@ -6,7 +6,9 @@ "source": [ "# Quickstart Tutorial\n", "\n", - "This tutorial will go over general `qcfractal` usage to give a feel for the ecosystem. In this tutorial, we employ Snowflake, a simple QCFractal stack which runs on a local machine for demonstration and exploration purposes.\n", + "This tutorial will go over general `qcfractal` usage to give a feel for the ecosystem. \n", + "In this tutorial, we employ Snowflake, a simple QCFractal stack which runs on a local machine \n", + "for demonstration and exploration purposes.\n", "\n", "## Installation\n", "\n", @@ -27,6 +29,7 @@ "## Importing QCFractal\n", "\n", "First let us import two items from the ecosystem:\n", + "\n", " * `FractalSnowflakeHandler` - This is a `FractalServer` that is temporary and is used for trying out new things.\n", " * `qcfractal.interface` is the `qcportal` module, but if using `qcfractal` it is best to import it locally.\n", " \n", @@ -49,7 +52,8 @@ "source": [ "We can now build a temporary server which acts just like a normal server, but we have a bit more direct control of it.\n", "\n", - "**Warning!** All data is lost when this notebook shuts down! This is for demonstration purposes only!" + "**Warning!** All data is lost when this notebook shuts down! This is for demonstration purposes only!\n", + "For information about how to setup a permanent QCFractal server, see the [Setup Quickstart Guide](setup_quickstart.rst). " ] }, { @@ -253,7 +257,7 @@ } ], "source": [ - "options = {\n", + "spec = {\n", " \"keywords\": None,\n", " \"qc_spec\": {\n", " \"driver\": \"gradient\",\n", @@ -264,7 +268,7 @@ "}\n", "\n", "# Ask the server to compute a new computation\n", - "r = client.add_procedure(\"optimization\", \"geometric\", options, [mol])\n", + "r = client.add_procedure(\"optimization\", \"geometric\", spec, [mol])\n", "print(r)\n", "print(r.ids)" ] @@ -291,7 +295,7 @@ } ], "source": [ - "r2 = client.add_procedure(\"optimization\", \"geometric\", options, [mol])\n", + "r2 = client.add_procedure(\"optimization\", \"geometric\", spec, [mol])\n", "print(r)\n", "print(r.ids)" ] @@ -529,7 +533,7 @@ "source": [ "## Collections\n", "\n", - "Submitting individual procedures or single quantum chemistry tasks is not typically done as it becomes hard to track individual ``Tasks``. To help resolve this, ``Collections`` are different ways of organizing standard computations so that many tasks can be referenced in a more human-friendly way. In this particular case, we will be exploring an intermolecular potential dataset.\n", + "Submitting individual procedures or single quantum chemistry tasks is not typically done as it becomes hard to track individual tasks. To help resolve this, ``Collections`` are different ways of organizing standard computations so that many tasks can be referenced in a more human-friendly way. In this particular case, we will be exploring an intermolecular potential dataset.\n", "\n", "To begin, we will create a new dataset and add a few intermolecular interactions to it." ] @@ -1205,6 +1209,7 @@ "metadata": {}, "source": [ "This is only the very beginning of what you can do with QCFractal! Explore the documentation to learn more capabilities.\n", + "In particular, see the [next section](setup_quickstart.rst) for a quickstart guide on how to set up QCFractal in production.\n", "\n", "If you like the project, consider starring us on [GitHub](https://github.com/MolSSI/QCFractal). If you have any questions, join our [Slack](https://join.slack.com/t/qcdb/shared_invite/enQtNDIzNTQ2OTExODk0LWM3OTgxN2ExYTlkMTlkZjA0OTExZDlmNGRlY2M4NWJlNDlkZGQyYWUxOTJmMzc3M2VlYzZjMjgxMDRkYzFmOTE) channel." ] @@ -1227,8 +1232,17 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.2" + }, + "pycharm": { + "stem_cell": { + "cell_type": "raw", + "source": [], + "metadata": { + "collapsed": false + } + } } }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/qcfractal/snowflake.py b/qcfractal/snowflake.py index 8d491ec9c..b176157c0 100644 --- a/qcfractal/snowflake.py +++ b/qcfractal/snowflake.py @@ -133,7 +133,9 @@ def __init__(self, query_limit=int(1.e6)) if self._storage: - self.logger.warning("Warning! This is a temporary instance, data will be lost upon shutdown.") + self.logger.warning("Warning! This is a temporary instance, data will be lost upon shutdown. " + "For information about how to set up a permanent QCFractal instance, see " + "http://docs.qcarchive.molssi.org/projects/qcfractal/en/latest/setup_quickstart.html") if start_server: self.start(start_loop=False) From 6f109ce96723fb74725358306b7a405a0ee4c32a Mon Sep 17 00:00:00 2001 From: Matthew Welborn Date: Thu, 15 Aug 2019 08:41:54 -0400 Subject: [PATCH 2/3] setup quickstart: spell check and PR suggested edits --- docs/qcfractal/source/setup_quickstart.rst | 28 +++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/qcfractal/source/setup_quickstart.rst b/docs/qcfractal/source/setup_quickstart.rst index fdc1ea68e..80f9abf8b 100644 --- a/docs/qcfractal/source/setup_quickstart.rst +++ b/docs/qcfractal/source/setup_quickstart.rst @@ -3,11 +3,11 @@ Setup Quickstart QCFractal comprises two components: -1. The :term:`Server` (``qcfractal-server``), which accepts compute and data queuries and maintains a database of :term:`tasks ` and results. The :term:`Server` should be run continuously on a persistant machine. +1. The :term:`Server` (``qcfractal-server``), which accepts compute and data queries and maintains a database of :term:`tasks ` and results. The :term:`Server` should be run continuously on a persistent machine. 2. One or more :term:`Managers ` (``qcfractal-manager``). The :term:`Managers ` pull work from the :term:`Server`, use attached compute resources to complete the work, and report results back to the server. :term:`Managers ` may be turned off and on at any time. :term:`Managers ` connect to compute resource through :term:`Adapters `. -In the :doc:`Quickstart Tutorial `, the above components were comined within a python envionment using ``FractalSnowflake``. -In general, the :term:`Server` and :term:`Manager(s) ` are run seapately, on separate machines. +In the :doc:`Quickstart Tutorial `, the above components were combined within a python environment using ``FractalSnowflake``. +In general, the :term:`Server` and :term:`Manager(s) ` are run separately in different process, often on different machines. For detailed information about the relationship between :term:`Server` and :term:`Manager`, see :doc:`managers`. @@ -46,7 +46,7 @@ The table below lists some common use cases for QCFractal: - Parsl QCFractal is highly adaptable and is not limited to the above use cases. -For example, it possible to mix local, cluster, supercomputer, and cloud :term:`Managers `. +For example, it possible to mix local, cluster, supercomputer, and cloud :term:`Managers ` simultaneously. In addition, a cloud instance may provide a good option for running ``qcfractal-server`` when a persistent web-exposed server is not otherwise available. Quickstart Setups @@ -75,7 +75,7 @@ Next, start the :term:`Server` and ProcessPoolExecutor :term:`Manager`:: The second command starts ``qcfractal-server`` in the background. It also starts one :term:`Worker` which will pull :term:`tasks ` from the :term:`Server` and run them. -Test if the everything is setup by running a Hartee-Fock calculation a single hydrogen molecule, +Test if everything is setup by running a Hartree-Fock calculation a single hydrogen molecule, as in the :doc:`quickstart` (note this requires ``psi4``): .. code-block:: python @@ -83,11 +83,13 @@ as in the :doc:`quickstart` (note this requires ``psi4``): python >>> import qcfractal.interface as ptl + # Note that server TLS verification is turned off (verify=False) since all components are run locally. >>> client = ptl.FractalClient(address="localhost:7777", verify=False) >>> mol = ptl.Molecule(symbols=["H", "H"], geometry=[0, 0, 0, 0, 5, 0]) >>> mol_id = client.add_molecules([mol])[0] >>> r = client.add_compute("psi4", "HF", "STO-3G", "energy", None, [mol_id]) + # Wait a minute for the job to complete >>> proc = client.query_procedures(id=r.ids)[0] >>> print(proc) @@ -147,7 +149,7 @@ Finally, start the :term:`Manager` in the background on the cluster head node:: Note that TLS certificate verification is disabled (``--verify=False``) because the :term:`Manager` and :term:`Server` are both run on the head node. -Test if the everything is setup by running a Hartee-Fock calculation a single hydrogen molecule, +Test if everything is setup by running a Hartree-Fock calculation a single hydrogen molecule, as in the :doc:`quickstart` (note this requires ``psi4``): .. code-block:: python @@ -155,11 +157,13 @@ as in the :doc:`quickstart` (note this requires ``psi4``): python >>> import qcfractal.interface as ptl + # Note that server TLS verification is turned off (verify=False) since all components are run locally. >>> client = ptl.FractalClient(address="localhost:7777", verify=False) >>> mol = ptl.Molecule(symbols=["H", "H"], geometry=[0, 0, 0, 0, 5, 0]) >>> mol_id = client.add_molecules([mol])[0] >>> r = client.add_compute("psi4", "HF", "STO-3G", "energy", None, [mol_id]) + # Wait a minute for the job to complete >>> proc = client.query_procedures(id=r.ids)[0] >>> print(proc) @@ -174,18 +178,18 @@ Shared Clusters, Supercomputers, and Multiple Clusters ++++++++++++++++++++++++++++++++++++++++++++++++++++++ This quickstart guide addresses QCFractal setup on one or more shared cluster(s). -The :term:`Server` should be set up on a persistant server for which you have permission to expose ports. +The :term:`Server` should be set up on a persistent server for which you have permission to expose ports. For example, this may be a dedicated webserver, the head node of a private cluster, or a cloud instance. The :term:`Manager` should be set up on each shared cluster. In most cases, the :term:`Manager` may be run on the head node; contact your system administrator if you are unsure. This guide requires `Parsl `_ to be installed for the :term:`Manager`. It may be installed with ``pip``. -Begin by initializing the :term:`Server` on your persistant server:: +Begin by initializing the :term:`Server` on your persistent server:: qcfractal-server init -The QCFractal server recieves connections from :term:`Managers ` and clients on TCP port 7777. +The QCFractal server receives connections from :term:`Managers ` and clients on TCP port 7777. You may optionally specify the ``--port`` option to choose a custom port. You may need to configure your firewall to allow access to this port. @@ -201,7 +205,7 @@ Start the :term:`Server`:: nohup qcfractal-server start & -You may optionally provide a TLS cerficiate to enable host verification for the :term:`Server` +You may optionally provide a TLS certificate to enable host verification for the :term:`Server` using the ``--tls-cert`` and ``--tls-key`` options. If a TLS certificate is not provided, communications with the server will still be encrypted, but host verification will be unavailable @@ -257,7 +261,7 @@ Finally, start the :term:`Manager` in the background on each cluster head node:: If you did not specify a TLS certificate in the ``qcfractal-server start`` step, you will additionally need to specify ``--verify False`` in the above command. -Test if the everything is setup by running a Hartee-Fock calculation a single hydrogen molecule, +Test if everything is setup by running a Hartree-Fock calculation a single hydrogen molecule, as in the :doc:`quickstart` (note this requires ``psi4`` to be installed on at least one compute resource). This test may be run from any machine. @@ -267,12 +271,14 @@ This test may be run from any machine. python >>> import qcfractal.interface as ptl + # Note that server TLS verification may need to be turned off if (verify=False). # Note that the Server URL and the password for user will need to be filled in. >>> client = ptl.FractalClient(address="URL:Port", username="user", password="***") >>> mol = ptl.Molecule(symbols=["H", "H"], geometry=[0, 0, 0, 0, 5, 0]) >>> mol_id = client.add_molecules([mol])[0] >>> r = client.add_compute("psi4", "HF", "STO-3G", "energy", None, [mol_id]) + # Wait a minute for the job to complete >>> proc = client.query_procedures(id=r.ids)[0] >>> print(proc) From a2b4c8a625ebb6e790e98f686264384fffcb736a Mon Sep 17 00:00:00 2001 From: Matthew Welborn Date: Thu, 15 Aug 2019 17:35:37 -0400 Subject: [PATCH 3/3] adds API links to quickstart notebook --- docs/qcfractal/source/quickstart.ipynb | 29 ++++++++++++++++++-------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/qcfractal/source/quickstart.ipynb b/docs/qcfractal/source/quickstart.ipynb index 1ffaa6d72..3779de9d8 100644 --- a/docs/qcfractal/source/quickstart.ipynb +++ b/docs/qcfractal/source/quickstart.ipynb @@ -6,7 +6,7 @@ "source": [ "# Quickstart Tutorial\n", "\n", - "This tutorial will go over general `qcfractal` usage to give a feel for the ecosystem. \n", + "This tutorial will go over general QCFractal usage to give a feel for the ecosystem. \n", "In this tutorial, we employ Snowflake, a simple QCFractal stack which runs on a local machine \n", "for demonstration and exploration purposes.\n", "\n", @@ -30,8 +30,8 @@ "\n", "First let us import two items from the ecosystem:\n", "\n", - " * `FractalSnowflakeHandler` - This is a `FractalServer` that is temporary and is used for trying out new things.\n", - " * `qcfractal.interface` is the `qcportal` module, but if using `qcfractal` it is best to import it locally.\n", + " * [FractalSnowflakeHandler](http://docs.qcarchive.molssi.org/projects/qcfractal/en/latest/api/qcfractal.FractalSnowflakeHandler.html?highlight=FractalSnowflakeHandler) - This is a [FractalServer](https://qcarchivetutorials.readthedocs.io/projects/qcfractal/en/latest/api/qcfractal.FractalServer.html) that is temporary and is used for trying out new things.\n", + " * `qcfractal.interface` is the [QCPortal](https://github.com/MolSSI/QCPortal) module, but if using QCFractal it is best to import it locally.\n", " \n", "Typically we alias `qcportal` as `ptl`. We will do the same for `qcfractal.interface` so that the code can be used anywhere." ] @@ -89,7 +89,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can then build a typical `FractalClient` to automatically connect to this server using the `client()` helper command. Note that the server names and addresses are identical in both the server and client." + "We can then build a typical [FractalClient](http://docs.qcarchive.molssi.org/projects/qcportal/en/latest/client.html?highlight=fractalclient#portal-client) \n", + "to automatically connect to this server using the [client()](http://docs.qcarchive.molssi.org/projects/qcfractal/en/latest/api/qcfractal.FractalSnowflakeHandler.html?highlight=FractalSnowflakeHandler#qcfractal.FractalSnowflakeHandler.client) helper command. \n", + "Note that the server names and addresses are identical in both the server and client." ] }, { @@ -128,7 +130,10 @@ "source": [ "## Adding and Querying data\n", "\n", - "A server starts with no data, so let's add some! We can do this by adding a water molecule at a poor geometry from XYZ coordinates. Note that all internal QCFractal values are stored and used in atomic units; whereas, the standard `Molecule.from_data` assumes an input of Angstroms. We can switch this back to Bohr by adding a `units` command in the text string. " + "A server starts with no data, so let's add some! We can do this by adding a water molecule at a poor geometry from XYZ coordinates. \n", + "Note that all internal QCFractal values are stored and used in atomic units; \n", + "whereas, the standard [Molecule.from_data()](http://docs.qcarchive.molssi.org/projects/qcelemental/en/latest/molecule.html?highlight=from_data#creation) assumes an input of Angstroms. \n", + "We can switch this back to Bohr by adding a `units` command in the text string. " ] }, { @@ -239,7 +244,11 @@ "\n", "We originally installed `psi4` and `geometric`, so we can use these programs to perform a geometry optimization. In QCFractal, we call a geometry optimization a `procedure`, where `procedure` is a generic term for a higher level operation that will run multiple individual quantum chemistry energy, gradient, or Hessian evaluations. Other `procedure` examples are finite-difference computations, n-body computations, and torsiondrives.\n", "\n", - "We provide a JSON-like input to the `client.add_procedure` command to specify the method, basis, and program to be used. The `qc_spec` field is used in all procedures to determine the underlying quantum chemistry method behind the individual procedure. In this way, we can use any program or method that returns a energy or gradient quantity to run our geometry optimization!" + "We provide a JSON-like input to the [client.add_procedure()](http://docs.qcarchive.molssi.org/projects/qcportal/en/latest/client-api.html?highlight=add_procedure#qcportal.FractalClient.add_procedure)\n", + " command to specify the method, basis, and program to be used. \n", + "The `qc_spec` field is used in all procedures to determine the underlying quantum chemistry method behind the individual procedure.\n", + "In this way, we can use any program or method that returns a energy or gradient quantity to run our geometry optimization!\n", + "(See also [add_compute()](http://docs.qcarchive.molssi.org/projects/qcportal/en/latest/client-api.html?highlight=add_procedure#qcportal.FractalClient.add_compute).)" ] }, { @@ -277,7 +286,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can see that we submitted a single task to be evaluated and the server has not seen this particular procedure before. The `ids` field returns the unique `id` of the procedure. Different procedures will always have a unique `id`, while identical procedures will always return the same `id`. We can submit the same procedure again to see this effect:" + "We can see that we submitted a single task to be evaluated and the server has not seen this particular procedure before. \n", + "The `ids` field returns the unique `id` of the procedure. Different procedures will always have a unique `id`, while identical procedures will always return the same `id`. \n", + "We can submit the same procedure again to see this effect:" ] }, { @@ -306,7 +317,7 @@ "source": [ "## Querying Procedures\n", "\n", - "Once a task is submitted, it will be placed in the compute queue and evaluated. In this particular case the `FractalSnowflakeHandler` uses your local hardware to evaluate these jobs. We recommend avoiding large tasks!\n", + "Once a task is submitted, it will be placed in the compute queue and evaluated. In this particular case the [FractalSnowflakeHandler](http://docs.qcarchive.molssi.org/projects/qcfractal/en/latest/api/qcfractal.FractalSnowflakeHandler.html?highlight=FractalSnowflakeHandler) uses your local hardware to evaluate these jobs. We recommend avoiding large tasks!\n", "\n", "In general, the server can handle anywhere between laptop-scale resources to many hundreds of thousands of concurrent cores at many physical locations. The amount of resources to connect is up to you and the amount of compute that you require.\n", "\n", @@ -338,7 +349,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This `OptimizationRecord` object has many different fields attached to it so that all quantities involved in the computation can be explored. For this example, let us pull the final molecule (optimized structure) and inspect the physical dimensions.\n", + "This [OptimizationRecord](http://docs.qcarchive.molssi.org/projects/qcportal/en/latest/record-api.html?highlight=optimizationrecord#qcportal.models.OptimizationRecord) object has many different fields attached to it so that all quantities involved in the computation can be explored. For this example, let us pull the final molecule (optimized structure) and inspect the physical dimensions.\n", "\n", "Note: if the status does not say `COMPLETE`, these fields will not be available. Try querying the procedure again in a few seconds to see if the task completed in the background." ]