Merge pull request #6 from sfiligoi/update_240718

Properly document TOROIDALS_PER_PROC and COLLISION_PRECISION_MODE

docs/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 711f83b3aa3bbab10fa381c061ae4e28
+config: 556eb8091613b199f169c4c376f2cb96
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/cgyro.html

@@ -7,7 +7,7 @@
   <title>CGYRO &mdash; GACODE</title>
       <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
       <link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=19f00094" />
-      <link rel="stylesheet" type="text/css" href="_static/contentui.css?v=4cf5b151" />
+      <link rel="stylesheet" type="text/css" href="_static/contentui.css" />
 
 
     <link rel="shortcut icon" href="_static/favicon.ico"/>
@@ -20,7 +20,7 @@
         <script src="_static/documentation_options.js?v=5929fcd5"></script>
         <script src="_static/doctools.js?v=9a2dae69"></script>
         <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
-        <script src="_static/contentui.js?v=9ee86694"></script>
+        <script src="_static/contentui.js"></script>
         <script async="async" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
     <script src="_static/js/theme.js"></script>
     <link rel="index" title="Index" href="genindex.html" />

docs/cgyro/cgyro_list.html

@@ -293,28 +293,18 @@ <h1>Alphabetical list for input.cgyro<a class="headerlink" href="#alphabetical-l
 <section id="collision-precision-mode">
 <span id="cgyro-collision-precision-mode"></span><h2>COLLISION_PRECISION_MODE<a class="headerlink" href="#collision-precision-mode" title="Link to this heading"></a></h2>
 <p><strong>Definition</strong></p>
-<p>Control the precision of the Sugama cmat constants.</p>
+<p>Control the precision of the cmat constants.</p>
 <p><strong>Choices</strong></p>
 <ul class="simple">
-<li><p>COLLISION_PRECISION_MODE = 0: Full 64-bit precision</p></li>
-<li><p>COLLISION_PRECISION_MODE = 1: Use lower-precision 32-bit precision for off-diagonal element.</p></li>
+<li><p>COLLISION_PRECISION_MODE = 0: Full 64-bit precision.</p></li>
+<li><p>COLLISION_PRECISION_MODE = 1: Heuristics-based lower-precision setup, slightly better than 32-bit precision.</p></li>
+<li><p>COLLISION_PRECISION_MODE = 32: Reduced 32-bit precision.</p></li>
+<li><p>COLLISION_PRECISION_MODE = 64: Full 64-bit precision. (Same as 0)</p></li>
 </ul>
 <p><strong>Comments</strong></p>
 <ul class="simple">
 <li><p>DEFAULT = 0</p></li>
-<li><p>Only active for <a class="reference internal" href="#cgyro-collision-model"><span class="std std-ref">COLLISION_MODEL</span></a> =4</p></li>
-<li><p>Selecting COLLISION_PRECISION_MODE = 1 results in significant memory saving and negligible precision loss in most use-cases.</p></li>
-</ul>
-<hr class="docutils" />
-</section>
-<section id="collision-full-stripes">
-<span id="cgyro-collision-full-stripes"></span><h2>COLLISION_FULL_STRIPES<a class="headerlink" href="#collision-full-stripes" title="Link to this heading"></a></h2>
-<p><strong>Definition</strong></p>
-<p>Number of full 64-bit stripes to use close to the diagonal in low-precision mode.</p>
-<p><strong>Comments</strong></p>
-<ul class="simple">
-<li><p>DEFAULT = 8</p></li>
-<li><p>Only active for <a class="reference internal" href="#cgyro-collision-model"><span class="std std-ref">COLLISION_MODEL</span></a> =4 and <a class="reference internal" href="#cgyro-collision-precision-mode"><span class="std std-ref">COLLISION_PRECISION_MODE</span></a> =1</p></li>
+<li><p>Selecting COLLISION_PRECISION_MODE 1 and 32 results in significant memory saving and negligible precision loss in most use-cases.</p></li>
 </ul>
 <hr class="docutils" />
 </section>
@@ -852,6 +842,7 @@ <h1>Alphabetical list for input.cgyro<a class="headerlink" href="#alphabetical-l
 <p><strong>Comments</strong></p>
 <ul class="simple">
 <li><p>Together with <a class="reference internal" href="#cgyro-ky"><span class="std std-ref">KY</span></a>, this controls the toroidal resolution.</p></li>
+<li><p>Together with <a class="reference internal" href="#cgyro-toroidals-per-proc"><span class="std std-ref">TOROIDALS_PER_PROC</span></a>, this controls the number of MPI processes needed.</p></li>
 </ul>
 <hr class="docutils" />
 </section>
@@ -1167,6 +1158,18 @@ <h1>Alphabetical list for input.cgyro<a class="headerlink" href="#alphabetical-l
 </ul>
 <hr class="docutils" />
 </section>
+<section id="toroidals-per-proc">
+<span id="cgyro-toroidals-per-proc"></span><h2>TOROIDALS_PER_PROC<a class="headerlink" href="#toroidals-per-proc" title="Link to this heading"></a></h2>
+<p><strong>Definition</strong></p>
+<p>Number of toroidal harmonics to process in a single MPI process.</p>
+<p><strong>Comments</strong></p>
+<ul class="simple">
+<li><p>DEFAULT: 1</p></li>
+<li><p>Must be a divisor of <a class="reference internal" href="#cgyro-n-toroidal"><span class="std std-ref">N_TOROIDAL</span></a>.</p></li>
+<li><p>A higher number typically results in faster simulation on low node count.</p></li>
+</ul>
+<hr class="docutils" />
+</section>
 <section id="udsymmetry-flag">
 <span id="cgyro-udsymmetry-flag"></span><h2>UDSYMMETRY_FLAG<a class="headerlink" href="#udsymmetry-flag" title="Link to this heading"></a></h2>
 <p><strong>Definition</strong></p>

docs/cgyro/cgyro_table.html

@@ -110,6 +110,7 @@ <h1>Tabular list for input.cgyro<a class="headerlink" href="#tabular-list-for-in
 <li><p><a class="reference internal" href="#tab-rotation"><span class="std std-ref">Rotation physics</span></a></p></li>
 <li><p><a class="reference internal" href="#tab-global"><span class="std std-ref">Global-spectral parameters</span></a></p></li>
 <li><p><a class="reference internal" href="#tab-output"><span class="std std-ref">Output file control</span></a></p></li>
+<li><p><a class="reference internal" href="#tab-optimization"><span class="std std-ref">Optimization related parameters</span></a></p></li>
 </ul>
 <span id="tab-shape"></span><table class="docutils align-default" id="id1">
 <caption><span class="caption-text"><strong>Plasma shape/geometry</strong></span><a class="headerlink" href="#id1" title="Link to this table"></a></caption>
@@ -421,11 +422,7 @@ <h1>Tabular list for input.cgyro<a class="headerlink" href="#tabular-list-for-in
 <td><p>Binormal spectral upwind order</p></td>
 <td><p>3</p></td>
 </tr>
-<tr class="row-even"><td><p><a class="reference internal" href="cgyro_list.html#cgyro-velocity-order"><span class="std std-ref">VELOCITY_ORDER</span></a></p></td>
-<td><p>What internal velocity order to use</p></td>
-<td><p>1</p></td>
-</tr>
-<tr class="row-odd"><td><p><a class="reference internal" href="cgyro_list.html#cgyro-upwind-single-flag"><span class="std std-ref">UPWIND_SINGLE_FLAG</span></a></p></td>
+<tr class="row-even"><td><p><a class="reference internal" href="cgyro_list.html#cgyro-upwind-single-flag"><span class="std std-ref">UPWIND_SINGLE_FLAG</span></a></p></td>
 <td><p>Use reduced precision communication</p></td>
 <td><p>0</p></td>
 </tr>
@@ -565,10 +562,6 @@ <h1>Tabular list for input.cgyro<a class="headerlink" href="#tabular-list-for-in
 <td><p>Reduce Sugama memory use</p></td>
 <td><p>0</p></td>
 </tr>
-<tr class="row-even"><td><p><a class="reference internal" href="cgyro_list.html#cgyro-gpu-bigmem-flag"><span class="std std-ref">GPU_BIGMEM_FLAG</span></a></p></td>
-<td><p>Enable GPU offload</p></td>
-<td><p>1</p></td>
-</tr>
 </tbody>
 </table>
 <span id="tab-rotation"></span><table class="docutils align-default" id="id10">
@@ -671,6 +664,38 @@ <h1>Tabular list for input.cgyro<a class="headerlink" href="#tabular-list-for-in
 </tr>
 </tbody>
 </table>
+<span id="tab-optimization"></span><table class="docutils align-default" id="id13">
+<caption><span class="caption-text"><strong>Optimization related parameters</strong></span><a class="headerlink" href="#id13" title="Link to this table"></a></caption>
+<colgroup>
+<col style="width: 30.2%" />
+<col style="width: 58.1%" />
+<col style="width: 11.6%" />
+</colgroup>
+<thead>
+<tr class="row-odd"><th class="head"><p>input.cgyro parameter</p></th>
+<th class="head"><p>Short description</p></th>
+<th class="head"><p>Default</p></th>
+</tr>
+</thead>
+<tbody>
+<tr class="row-even"><td><p><a class="reference internal" href="cgyro_list.html#cgyro-toroidals-per-proc"><span class="std std-ref">TOROIDALS_PER_PROC</span></a></p></td>
+<td><p>How many toroidal harmonics per MPI process</p></td>
+<td><p>1</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="cgyro_list.html#cgyro-mpi-rank-order"><span class="std std-ref">MPI_RANK_ORDER</span></a></p></td>
+<td><p>Relative ordering of MPI ranks</p></td>
+<td><p>2</p></td>
+</tr>
+<tr class="row-even"><td><p><a class="reference internal" href="cgyro_list.html#cgyro-velocity-order"><span class="std std-ref">VELOCITY_ORDER</span></a></p></td>
+<td><p>What internal velocity order to use</p></td>
+<td><p>1</p></td>
+</tr>
+<tr class="row-odd"><td><p><a class="reference internal" href="cgyro_list.html#cgyro-gpu-bigmem-flag"><span class="std std-ref">GPU_BIGMEM_FLAG</span></a></p></td>
+<td><p>Enable GPU offload when possible</p></td>
+<td><p>1</p></td>
+</tr>
+</tbody>
+</table>
 </section>
 
 

docs/genindex.html

@@ -6,7 +6,7 @@
   <title>Index &mdash; GACODE</title>
       <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
       <link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=19f00094" />
-      <link rel="stylesheet" type="text/css" href="_static/contentui.css?v=4cf5b151" />
+      <link rel="stylesheet" type="text/css" href="_static/contentui.css" />
 
 
     <link rel="shortcut icon" href="_static/favicon.ico"/>
@@ -19,7 +19,7 @@
         <script src="_static/documentation_options.js?v=5929fcd5"></script>
         <script src="_static/doctools.js?v=9a2dae69"></script>
         <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
-        <script src="_static/contentui.js?v=9ee86694"></script>
+        <script src="_static/contentui.js"></script>
     <script src="_static/js/theme.js"></script>
     <link rel="index" title="Index" href="#" />
     <link rel="search" title="Search" href="search.html" /> 

docs/index.html

@@ -7,7 +7,7 @@
   <title>The General Atomics GACODE Suite &mdash; GACODE</title>
       <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
       <link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=19f00094" />
-      <link rel="stylesheet" type="text/css" href="_static/contentui.css?v=4cf5b151" />
+      <link rel="stylesheet" type="text/css" href="_static/contentui.css" />
 
 
     <link rel="shortcut icon" href="_static/favicon.ico"/>
@@ -20,7 +20,7 @@
         <script src="_static/documentation_options.js?v=5929fcd5"></script>
         <script src="_static/doctools.js?v=9a2dae69"></script>
         <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
-        <script src="_static/contentui.js?v=9ee86694"></script>
+        <script src="_static/contentui.js"></script>
         <script async="async" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
     <script src="_static/js/theme.js"></script>
     <link rel="index" title="Index" href="genindex.html" />

docs/search.html

@@ -6,7 +6,7 @@
   <title>Search &mdash; GACODE</title>
       <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
       <link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=19f00094" />
-      <link rel="stylesheet" type="text/css" href="_static/contentui.css?v=4cf5b151" />
+      <link rel="stylesheet" type="text/css" href="_static/contentui.css" />
 
 
     <link rel="shortcut icon" href="_static/favicon.ico"/>
@@ -20,7 +20,7 @@
         <script src="_static/documentation_options.js?v=5929fcd5"></script>
         <script src="_static/doctools.js?v=9a2dae69"></script>
         <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
-        <script src="_static/contentui.js?v=9ee86694"></script>
+        <script src="_static/contentui.js"></script>
     <script src="_static/js/theme.js"></script>
     <script src="_static/searchtools.js"></script>
     <script src="_static/language_data.js"></script>

html/src/cgyro/cgyro_list.rst

@@ -277,34 +277,19 @@ COLLISION_PRECISION_MODE
 
 **Definition**
 
-Control the precision of the Sugama cmat constants.
+Control the precision of the cmat constants.
 
 **Choices**
 
-- COLLISION_PRECISION_MODE = 0: Full 64-bit precision
-- COLLISION_PRECISION_MODE = 1: Use lower-precision 32-bit precision for off-diagonal element.
+- COLLISION_PRECISION_MODE = 0: Full 64-bit precision.
+- COLLISION_PRECISION_MODE = 1: Heuristics-based lower-precision setup, slightly better than 32-bit precision.
+- COLLISION_PRECISION_MODE = 32: Reduced 32-bit precision.
+- COLLISION_PRECISION_MODE = 64: Full 64-bit precision. (Same as 0)
 
 **Comments**
 
 - DEFAULT = 0
-- Only active for :ref:`cgyro_collision_model` =4
-- Selecting COLLISION_PRECISION_MODE = 1 results in significant memory saving and negligible precision loss in most use-cases.
-
-----
-
-.. _cgyro_collision_full_stripes:
-
-COLLISION_FULL_STRIPES
-----------------------
-
-**Definition**
-
-Number of full 64-bit stripes to use close to the diagonal in low-precision mode.
-
-**Comments**
-
-- DEFAULT = 8
-- Only active for :ref:`cgyro_collision_model` =4 and :ref:`cgyro_collision_precision_mode` =1
+- Selecting COLLISION_PRECISION_MODE 1 and 32 results in significant memory saving and negligible precision loss in most use-cases.
 
 ----
 
@@ -1066,6 +1051,7 @@ Number of toroidal harmonics (binormal Fourier modes).
 **Comments**
 
 - Together with :ref:`cgyro_ky`, this controls the toroidal resolution.
+- Together with :ref:`cgyro_toroidals_per_proc`, this controls the number of MPI processes needed.
 
 ----
 
@@ -1513,6 +1499,23 @@ The number of poloidal points for output of large fluctuation moment arrays.
 - Value should be an integer divisor of :ref:`cgyro_n_theta`
 
 
+----
+
+.. _cgyro_toroidals_per_proc:
+
+TOROIDALS_PER_PROC
+------------------
+
+**Definition**
+
+Number of toroidal harmonics to process in a single MPI process.
+
+**Comments**
+
+- DEFAULT: 1
+- Must be a divisor of :ref:`cgyro_n_toroidal`.
+- A higher number typically results in faster simulation on low node count.
+
 ----
 
 .. _cgyro_udsymmetry_flag:

html/src/cgyro/cgyro_table.rst

@@ -13,6 +13,7 @@ Tabular list for input.cgyro
 * :ref:`tab-rotation`
 * :ref:`tab-global`
 * :ref:`tab-output`
+* :ref:`tab-optimization`
 
 .. _tab-shape:
 
@@ -109,7 +110,6 @@ Tabular list for input.cgyro
    ":ref:`cgyro_nup_radial`", Radial spectral upwind order,3
    ":ref:`cgyro_nup_theta`", Poloidal upwind order,3
    ":ref:`cgyro_nup_alpha`", Binormal spectral upwind order,3
-   ":ref:`cgyro_velocity_order`", What internal velocity order to use,1
    ":ref:`cgyro_upwind_single_flag`", Use reduced precision communication,0
 
 .. _tab-time:
@@ -154,7 +154,6 @@ Tabular list for input.cgyro
    ":ref:`cgyro_collision_ene_diffusion`", Toggle energy diffusion,1
    ":ref:`cgyro_collision_kperp`", Toggle so-called FLR term, 0
    ":ref:`cgyro_collision_precision_mode`", Reduce Sugama memory use, 0
-   ":ref:`cgyro_gpu_bigmem_flag`", Enable GPU offload, 1
 
 .. _tab-rotation:
 
@@ -189,3 +188,15 @@ Tabular list for input.cgyro
    ":ref:`cgyro_moment_print_flag`", Output of density and energy moments, 0
    ":ref:`cgyro_gflux_print_flag`", Output of global flux profiles, 0
    ":ref:`cgyro_h_print_flag`", Output of distribution (single-mode only), 0
+
+.. _tab-optimization:
+
+.. csv-table:: **Optimization related parameters**
+   :header: "input.cgyro parameter", "Short description", "Default"
+   :widths: 13, 25, 5
+
+   ":ref:`cgyro_toroidals_per_proc`", How many toroidal harmonics per MPI process,1
+   ":ref:`cgyro_mpi_rank_order`", Relative ordering of MPI ranks,2
+   ":ref:`cgyro_velocity_order`", What internal velocity order to use,1
+   ":ref:`cgyro_gpu_bigmem_flag`", Enable GPU offload when possible, 1
+