Merge pull request #349 from NREL/kill-docker-singularity

Remove Docker dependency

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
     name: Tests - Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v3
@@ -32,6 +32,12 @@ jobs:
           mkdir weather
           cd weather
           wget --quiet https://data.nrel.gov/system/files/156/BuildStock_TMY3_FIPS.zip
+      - name: Download and Install OpenStudio
+        run: |
+          wget -q https://github.com/NREL/OpenStudio/releases/download/v3.5.1/OpenStudio-3.5.1+22e1db7be5-Ubuntu-20.04.deb
+          sudo apt install -y ./OpenStudio-3.5.1+22e1db7be5-Ubuntu-20.04.deb
+          openstudio openstudio_version
+          which openstudio
       - name: Install buildstockbatch
         run: |
           cd buildstockbatch
@@ -42,41 +48,41 @@ jobs:
           cd buildstockbatch
           # stop the build if there are Python syntax errors or undefined names
           flake8 buildstockbatch --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 buildstockbatch --count --max-line-length=127 --statistics
+          # exit-zero treats all errors as warnings.
+          flake8 buildstockbatch --count --statistics --exit-zero
       - name: Run PyTest and Coverage
         run: |
           cd buildstockbatch
           pytest --junitxml=coverage/junit.xml --cov=buildstockbatch --cov-report=xml:coverage/coverage.xml --cov-report=html:coverage/htmlreport
       - name: Test Report
         uses: mikepenz/[email protected]
-        if: ${{ matrix.python-version == '3.10' }}
+        if: ${{ matrix.python-version == '3.11' }}
         with:
           report_paths: buildstockbatch/coverage/junit.xml
           check_name: Testing Report
           fail_on_failure: true
       - name: Save Coverage Report
         uses: actions/upload-artifact@v3
-        if: ${{ matrix.python-version == '3.10' }}
+        if: ${{ matrix.python-version == '3.11' }}
         with:
           name: coverage-report-html
           path: buildstockbatch/coverage/htmlreport/
       - name: Report coverage to PR
         uses: 5monkeys/cobertura-action@v13
-        if: ${{ matrix.python-version == '3.10' }}
+        if: ${{ matrix.python-version == '3.11' }}
         with:
           path: buildstockbatch/coverage/coverage.xml
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           minimum_coverage: 33
           fail_below_threshold: true
       - name: Build documentation
-        if: ${{ matrix.python-version == '3.10' }}
+        if: ${{ matrix.python-version == '3.11' }}
         run: |
           cd buildstockbatch/docs
           make html SPHINXOPTS="-W --keep-going -n"
       - name: Save Docs
         uses: actions/upload-artifact@v3
-        if: ${{ matrix.python-version == '3.10' }}
+        if: ${{ matrix.python-version == '3.11' }}
         with:
           name: documentation
           path: buildstockbatch/docs/_build/html/

.gitignore

@@ -14,3 +14,4 @@ buildstockbatch_crash_details.log
 coverage/
 .coverage
 build/
+.env

buildstockbatch/__init__.py

@@ -9,20 +9,20 @@
 Executing BuildStock projects on batch infrastructure
 ~~~~~~~~~~~~~~~~~~~~~
 BuildStockBatch is a simulation runtime library, written in Python, to allow researchers to execute the very large scale
-simulation sets required for BuildStock analyses. Basic Eagle usage:
+simulation sets required for ResStock and ComStock analyses. Basic Eagle usage:
 ```
    [user@loginN ~]$ module load conda
-   [user@loginN ~]$ source activate /shared-projects/buildstock/envs/buildstock-X.X
+   [user@loginN ~]$ source activate /shared-projects/buildstock/envs/buildstock-YYYY.MM.X
    [user@loginN ~]$ buildstock_eagle ~/buildstockbatch/project_resstock_national.yml
 ```
 ... or locally using Docker:
 ```
    user$ pyenv activate buildstockbatch
    user$ pip install -e ./buildstockbatch
-   user$ buildstock_docker -j -2 ~/buildstockbatch/project_resstock_multifamily.yml
+   user$ buildstock_local ~/buildstockbatch/project_resstock_multifamily.yml
 ```
 Other batch simulation methods may be supported in future. Please refer to the documentation for more
 details regarding these features, and configuration via the project yaml configuration documentation.
-:copyright: (c) 2018 by The Alliance for Sustainable Energy.
+:copyright: (c) 2023 by The Alliance for Sustainable Energy.
 :license: BSD-3, see LICENSE for more details.
 """

buildstockbatch/aws/aws.py

@@ -16,6 +16,7 @@
 from botocore.exceptions import ClientError
 import collections
 import csv
+import docker
 from fsspec.implementations.local import LocalFileSystem
 import gzip
 import hashlib
@@ -38,11 +39,10 @@
 import io
 import zipfile
 
-from buildstockbatch.localdocker import DockerBatchBase
-from buildstockbatch.base import ValidationError
+from buildstockbatch.base import ValidationError, BuildStockBatchBase
 from buildstockbatch.aws.awsbase import AwsJobBase
 from buildstockbatch import postprocessing
-from ..utils import log_error_details, get_project_configuration
+from buildstockbatch.utils import ContainerRuntime, log_error_details, get_project_configuration
 
 logger = logging.getLogger(__name__)
 
@@ -1658,6 +1658,29 @@ def clean(self):
         logger.info(f"Simple notifications topic {self.sns_state_machine_topic} deleted.")
 
 
+class DockerBatchBase(BuildStockBatchBase):
+
+    CONTAINER_RUNTIME = ContainerRuntime.DOCKER
+
+    def __init__(self, project_filename):
+        super().__init__(project_filename)
+
+        self.docker_client = docker.DockerClient.from_env()
+        try:
+            self.docker_client.ping()
+        except:  # noqa: E722 (allow bare except in this case because error can be a weird non-class Windows API error)
+            logger.error('The docker server did not respond, make sure Docker Desktop is started then retry.')
+            raise RuntimeError('The docker server did not respond, make sure Docker Desktop is started then retry.')
+
+    @staticmethod
+    def validate_project(project_file):
+        super(DockerBatchBase, DockerBatchBase).validate_project(project_file)
+
+    @property
+    def docker_image(self):
+        return 'nrel/openstudio:{}'.format(self.os_version)
+
+
 class AwsBatch(DockerBatchBase):
 
     def __init__(self, project_filename):

buildstockbatch/base.py

@@ -3,7 +3,7 @@
 """
 buildstockbatch.base
 ~~~~~~~~~~~~~~~
-This is the base class mixed into the deployment specific classes (i.e. eagle, localdocker)
+This is the base class mixed into the deployment specific classes (i.e. eagle, local)
 
 :author: Noel Merket
 :copyright: (c) 2018 by The Alliance for Sustainable Energy
@@ -20,6 +20,7 @@
 import re
 import requests
 import shutil
+import subprocess
 import tempfile
 import yamale
 import zipfile
@@ -91,6 +92,10 @@ def sampler(self):
         Sampler = self.get_sampler_class(self.cfg['sampler']['type'])
         return Sampler(self, **self.cfg['sampler'].get('args', {}))
 
+    @staticmethod
+    def openstudio_exe():
+        return os.environ.get("OPENSTUDIO_EXE", "openstudio")
+
     def path_rel_to_projectfile(self, x):
         return path_rel_to_file(self.project_filename, x)
 
@@ -196,7 +201,7 @@ def cleanup_sim_dir(sim_dir, dest_fs, simout_ts_dir, upgrade_id, building_id):
         schedules_filepath = ''
         if os.path.isdir(os.path.join(sim_dir, 'generated_files')):
             for file in os.listdir(os.path.join(sim_dir, 'generated_files')):
-                if file.endswith('schedules.csv'):
+                if re.match(r".*schedules.*\.csv", file):
                     schedules_filepath = os.path.join(sim_dir, 'generated_files', file)
 
         if os.path.isfile(timeseries_filepath):
@@ -249,21 +254,21 @@ def get_clean_column_name(x):
         if os.path.isdir(reports_dir):
             shutil.rmtree(reports_dir, ignore_errors=True)
 
-    @staticmethod
-    def validate_project(project_file):
-        assert BuildStockBatchBase.validate_project_schema(project_file)
-        assert BuildStockBatchBase.validate_sampler(project_file)
-        assert BuildStockBatchBase.validate_workflow_generator(project_file)
-        assert BuildStockBatchBase.validate_misc_constraints(project_file)
-        assert BuildStockBatchBase.validate_xor_nor_schema_keys(project_file)
-        assert BuildStockBatchBase.validate_reference_scenario(project_file)
-        assert BuildStockBatchBase.validate_options_lookup(project_file)
-        assert BuildStockBatchBase.validate_logic(project_file)
-        assert BuildStockBatchBase.validate_measure_references(project_file)
-        assert BuildStockBatchBase.validate_postprocessing_spec(project_file)
-        assert BuildStockBatchBase.validate_resstock_or_comstock_version(project_file)
-        assert BuildStockBatchBase.validate_openstudio_version(project_file)
-        assert BuildStockBatchBase.validate_number_of_options(project_file)
+    @classmethod
+    def validate_project(cls, project_file):
+        assert cls.validate_project_schema(project_file)
+        assert cls.validate_sampler(project_file)
+        assert cls.validate_workflow_generator(project_file)
+        assert cls.validate_misc_constraints(project_file)
+        assert cls.validate_xor_nor_schema_keys(project_file)
+        assert cls.validate_reference_scenario(project_file)
+        assert cls.validate_options_lookup(project_file)
+        assert cls.validate_logic(project_file)
+        assert cls.validate_measure_references(project_file)
+        assert cls.validate_postprocessing_spec(project_file)
+        assert cls.validate_resstock_or_comstock_version(project_file)
+        assert cls.validate_openstudio_version(project_file)
+        assert cls.validate_number_of_options(project_file)
         logger.info('Base Validation Successful')
         return True
 
@@ -275,6 +280,31 @@ def get_buildstock_dir(project_file, cfg):
         else:
             return os.path.abspath(os.path.join(os.path.dirname(project_file), buildstock_dir))
 
+    @classmethod
+    def validate_openstudio_path(cls, project_file):
+        cfg = get_project_configuration(project_file)
+        os_version = cfg.get('os_version', cls.DEFAULT_OS_VERSION)
+        os_sha = cfg.get('os_sha', cls.DEFAULT_OS_SHA)
+        try:
+            proc_out = subprocess.run(
+                [cls.openstudio_exe(), "openstudio_version"],
+                capture_output=True,
+                text=True
+            )
+        except FileNotFoundError:
+            raise ValidationError(f"Cannot find openstudio at `{cls.openstudio_exe()}`")
+        if proc_out.returncode != 0:
+            raise ValidationError(f"OpenStudio failed with the following error {proc_out.stderr}")
+        actual_os_version, actual_os_sha = proc_out.stdout.strip().split("+")
+        if os_version != actual_os_version:
+            raise ValidationError(f"OpenStudio version is {actual_os_version}, expected is {os_version}")
+        if os_sha != actual_os_sha:
+            raise ValidationError(
+                f"OpenStudio version is correct at {os_version}, but the shas don't match. "
+                "Got {actual_os_sha}, expected {os_sha}"
+            )
+        return True
+
     @staticmethod
     def validate_sampler(project_file):
         cfg = get_project_configuration(project_file)

buildstockbatch/eagle.py

@@ -71,9 +71,9 @@ def __init__(self, project_filename):
         logger.debug('Output directory = {}'.format(output_dir))
         weather_dir = self.weather_dir  # noqa E841
 
-    @staticmethod
-    def validate_project(project_file):
-        super(EagleBatch, EagleBatch).validate_project(project_file)
+    @classmethod
+    def validate_project(cls, project_file):
+        super(cls, cls).validate_project(project_file)
         # Eagle specific validation goes here
 
     @property