Merge pull request #168 from NREL/pp/gaps_integration

GAPs integration

requirements.txt

@@ -1,8 +1,7 @@
 matplotlib>=3.1
 NREL-rex>=0.2.84
 NREL-phygnn>=0.0.23
-NREL-rev<0.8.0
-NREL-gaps>=0.4.0
+NREL-gaps>=0.6.0
 NREL-farms>=1.0.4
 google-auth-oauthlib==0.5.3
 pytest>=5.2

sup3r/__init__.py

@@ -2,6 +2,7 @@
 """Super Resolving Renewable Energy Resource Data (SUP3R)"""
 import os
 from sup3r.version import __version__
+import sup3r.cli  # import sets up CLI commands
 
 __author__ = """Brandon Benton"""
 __email__ = "[email protected]"

sup3r/batch/batch_cli.py

@@ -5,12 +5,8 @@
 """
 import click
 
-from rex.utilities.utilities import get_class_properties
-
-from reV.config.batch import BatchConfig
-
 from sup3r import __version__
-from sup3r.batch.batch import BatchJob
+from gaps.batch import BatchJob
 
 
 @click.group()
@@ -24,14 +20,6 @@ def main(ctx, verbose):
     ctx.obj['VERBOSE'] = verbose
 
 
-@main.command()
-def valid_config_keys():
-    """
-    Echo the valid Batch config keys
-    """
-    click.echo(', '.join(get_class_properties(BatchConfig)))
-
-
 @main.command()
 @click.option('--config_file', '-c', required=True,
               type=click.Path(exists=True),
@@ -52,18 +40,18 @@ def valid_config_keys():
               help='Flag to turn on debug logging. Default is not verbose.')
 @click.pass_context
 def from_config(ctx, config_file, dry_run, cancel, delete, monitor_background,
-                verbose):
+                verbose=False):
     """Run Sup3r batch from a config file."""
-    verbose = any([verbose, ctx.obj['VERBOSE']])
+    ctx.ensure_object(dict)
+    ctx.obj['VERBOSE'] = verbose or ctx.obj.get('VERBOSE', False)
+    batch = BatchJob(config_file)
 
     if cancel:
-        BatchJob.cancel_all(config_file, verbose=verbose)
+        batch.cancel()
     elif delete:
-        BatchJob.delete_all(config_file, verbose=verbose)
+        batch.delete()
     else:
-        BatchJob.run(config_file, dry_run=dry_run,
-                     monitor_background=monitor_background,
-                     verbose=verbose)
+        batch.run(dry_run=dry_run, monitor_background=monitor_background)
 
 
 if __name__ == '__main__':

sup3r/bias/bias_calc.py

@@ -182,7 +182,7 @@ def get_node_cmd(cls, config):
             initialize the class and call run() on a single node.
         """
         import_str = 'import time;\n'
-        import_str += 'from sup3r.pipeline import Status;\n'
+        import_str += 'from gaps import Status;\n'
         import_str += 'from rex import init_logger;\n'
         import_str += f'from sup3r.bias.bias_calc import {cls.__name__};\n'
 

sup3r/bias/bias_calc_cli.py

@@ -10,7 +10,7 @@
 
 import sup3r.bias.bias_calc
 from sup3r.utilities import ModuleName
-from sup3r.utilities.cli import BaseCLI
+from sup3r.utilities.cli import AVAILABLE_HARDWARE_OPTIONS, BaseCLI
 from sup3r.version import __version__
 
 logger = logging.getLogger(__name__)
@@ -34,7 +34,7 @@ def main(ctx, verbose):
 @click.option('-v', '--verbose', is_flag=True,
               help='Flag to turn on debug logging. Default is not verbose.')
 @click.pass_context
-def from_config(ctx, config_file, verbose):
+def from_config(ctx, config_file, verbose=False, **__):
     """Run sup3r bias correction calculation from a config file."""
     config = BaseCLI.from_config_preflight(ModuleName.BIAS_CALC, ctx,
                                            config_file, verbose)
@@ -62,7 +62,7 @@ def from_config(ctx, config_file, verbose):
         cmd_log = '\n\t'.join(cmd.split('\n'))
         logger.debug(f'Running command:\n\t{cmd_log}')
 
-        if hardware_option.lower() in ('eagle', 'slurm'):
+        if hardware_option.lower() in AVAILABLE_HARDWARE_OPTIONS:
             kickoff_slurm_job(ctx, cmd, **exec_kwargs)
         else:
             kickoff_local_job(ctx, cmd)

sup3r/cli.py

@@ -5,7 +5,10 @@
 import click
 import logging
 
+from gaps import Pipeline
+
 from sup3r.version import __version__
+from sup3r.utilities import ModuleName
 from sup3r.pipeline.forward_pass_cli import from_config as fwp_cli
 from sup3r.solar.solar_cli import from_config as solar_cli
 from sup3r.preprocessing.data_extract_cli import from_config as dh_cli
@@ -14,7 +17,6 @@
 from sup3r.qa.visual_qa_cli import from_config as visual_qa_cli
 from sup3r.qa.stats_cli import from_config as stats_cli
 from sup3r.pipeline.pipeline_cli import from_config as pipe_cli
-from sup3r.pipeline.pipeline_cli import valid_config_keys as pipeline_keys
 from sup3r.batch.batch_cli import from_config as batch_cli
 from sup3r.bias.bias_calc_cli import from_config as bias_calc_cli
 from sup3r.utilities.regridder_cli import from_config as regrid_cli
@@ -114,18 +116,15 @@ def forward_pass(ctx, verbose):
                 },
             },
             "execution_control": {
-                "option": "local"
-            },
-            "execution_control_eagle": {
-                "option": "eagle",
+                "option": "kestrel",
                 "walltime": 4,
                 "alloc": "sup3r"
             }
         }
 
-    Note that the ``execution_control`` block will run the job locally, while
-    the ``execution_control_eagle`` block are kwargs that would be required to
-    distribute the job on multiple nodes on the NREL HPC.
+    Note that the ``execution_control`` block contains kwargs that would
+    be required to distribute the job on multiple nodes on the NREL HPC.
+    To run the job locally, use ``execution_control: {"option": "local"}``.
     """
     config_file = ctx.obj['CONFIG_FILE']
     verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -163,18 +162,15 @@ def solar(ctx, verbose):
             "fp_pattern": "./chunks/sup3r*.h5",
             "nsrdb_fp": "/datasets/NSRDB/current/nsrdb_2015.h5",
             "execution_control": {
-                "option": "local"
-            },
-            "execution_control_eagle": {
-                "option": "eagle",
+                "option": "kestrel",
                 "walltime": 4,
                 "alloc": "sup3r"
             }
         }
 
-    Note that the ``execution_control`` block will run the job locally, while
-    the ``execution_control_eagle`` block are kwargs that would be required to
-    distribute the job on multiple nodes on the NREL HPC.
+   Note that the ``execution_control`` block contains kwargs that would
+    be required to distribute the job on multiple nodes on the NREL HPC.
+    To run the job locally, use ``execution_control: {"option": "local"}``.
     """
     config_file = ctx.obj['CONFIG_FILE']
     verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -227,18 +223,15 @@ def bias_calc(ctx, verbose):
                  }
             ],
             "execution_control": {
-                "option": "local"
-            },
-            "execution_control_eagle": {
-                "option": "eagle",
+                "option": "kestrel",
                 "walltime": 4,
                 "alloc": "sup3r"
             }
         }
 
-    Note that the ``execution_control`` block will run the job locally, while
-    the ``execution_control_eagle`` block are kwargs that would be required to
-    distribute the job on multiple nodes on the NREL HPC.
+    Note that the ``execution_control`` block contains kwargs that would
+    be required to distribute the job on multiple nodes on the NREL HPC.
+    To run the job locally, use ``execution_control: {"option": "local"}``.
     """
     config_file = ctx.obj['CONFIG_FILE']
     verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -277,7 +270,7 @@ def data_extract(ctx, verbose):
         }
 
     Note that the ``execution_control`` has the same options as forward-pass
-    and you can set ``"option": "eagle"`` to run on the NREL HPC.
+    and you can set ``"option": "kestrel"`` to run on the NREL HPC.
     """
     config_file = ctx.obj['CONFIG_FILE']
     verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -316,7 +309,7 @@ def data_collect(ctx, verbose):
         }
 
     Note that the ``execution_control`` has the same options as forward-pass
-    and you can set ``"option": "eagle"`` to run on the NREL HPC.
+    and you can set ``"option": "kestrel"`` to run on the NREL HPC.
     """
     config_file = ctx.obj['CONFIG_FILE']
     verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -357,7 +350,7 @@ def qa(ctx, verbose):
         }
 
     Note that the ``execution_control`` has the same options as forward-pass
-    and you can set ``"option": "eagle"`` to run on the NREL HPC.
+    and you can set ``"option": "kestrel"`` to run on the NREL HPC.
     """
     config_file = ctx.obj['CONFIG_FILE']
     verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -397,7 +390,7 @@ def visual_qa(ctx, verbose):
         }
 
     Note that the ``execution_control`` has the same options as forward-pass
-    and you can set ``"option": "eagle"`` to run on the NREL HPC.
+    and you can set ``"option": "kestrel"`` to run on the NREL HPC.
     """
     config_file = ctx.obj['CONFIG_FILE']
     verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -441,7 +434,7 @@ def stats(ctx, verbose):
         }
 
     Note that the ``execution_control`` has the same options as forward-pass
-    and you can set ``"option": "eagle"`` to run on the NREL HPC.
+    and you can set ``"option": "kestrel"`` to run on the NREL HPC.
     """
     config_file = ctx.obj['CONFIG_FILE']
     verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -484,7 +477,7 @@ def regrid(ctx, verbose):
         }
 
     Note that the ``execution_control`` has the same options as forward-pass
-    and you can set ``"option": "eagle"`` to run on the NREL HPC.
+    and you can set ``"option": "kestrel"`` to run on the NREL HPC.
     """
     config_file = ctx.obj['CONFIG_FILE']
     verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -539,13 +532,6 @@ def pipeline(ctx, cancel, monitor, background, verbose):
                    monitor=monitor, background=background, verbose=verbose)
 
 
-@pipeline.command()
-@click.pass_context
-def valid_pipeline_keys(ctx):
-    """Print the valid pipeline config keys"""
-    ctx.invoke(pipeline_keys)
-
-
 @main.group(invoke_without_command=True)
 @click.option('--dry-run', is_flag=True,
               help='Flag to do a dry run (make batch dirs without running).')
@@ -598,6 +584,17 @@ def batch(ctx, dry_run, cancel, delete, monitor_background, verbose):
                    verbose=verbose)
 
 
+Pipeline.COMMANDS[ModuleName.FORWARD_PASS] = fwp_cli
+Pipeline.COMMANDS[ModuleName.SOLAR] = solar_cli
+Pipeline.COMMANDS[ModuleName.DATA_EXTRACT] = dh_cli
+Pipeline.COMMANDS[ModuleName.DATA_COLLECT] = dc_cli
+Pipeline.COMMANDS[ModuleName.QA] = qa_cli
+Pipeline.COMMANDS[ModuleName.VISUAL_QA] = visual_qa_cli
+Pipeline.COMMANDS[ModuleName.STATS] = stats_cli
+Pipeline.COMMANDS[ModuleName.BIAS_CALC] = bias_calc_cli
+Pipeline.COMMANDS[ModuleName.REGRID] = regrid_cli
+
+
 if __name__ == '__main__':
     try:
         main(obj={})

sup3r/pipeline/__init__.py

@@ -2,6 +2,3 @@
 """
 Sup3r data pipeline architecture.
 """
-from gaps.legacy import Status
-
-from .pipeline import Sup3rPipeline