hpc parameter+

.gitignore

@@ -4,3 +4,6 @@
 
 build/
 dist/
+venv/
+.vscode/settings.json
+processMeerKAT/workspace.code-workspace

README.md

@@ -16,8 +16,6 @@ This pipeline is designed to run on the Ilifu cluster, making use of SLURM and M
 
 ## 1. Setup the pipeline in your environment
 
-**Please note : These docs are for an upcoming release of the pipeline. At the time of writing (July 2022) please use the CASA6 branch of the pipeline, which is the pre-release version. The master branch will be updated at the time of release.**
-
 In order to use the `processMeerKAT.py` script, source the `setup.sh` file, which can be done on [ilifu](https://docs.ilifu.ac.za/#/) as
 
         source /idia/software/pipelines/master/setup.sh
@@ -67,6 +65,6 @@ Starting with v1.1 of the processMeerKAT pipeline, the default behaviour is to s
 
 2. **Logs in the top level directory** : Logs in the top level directory (*i.e.,* the directory where the pipeline was launched) correspond to the scripts in the `precal_scripts` and `postcal_scripts` variables in the config file. These scripts are run from the top level before and after calibration respectively. By default these correspond to the scripts to calculate the reference antenna (if enabled), partition the data into SPWs, and concat the individual SPWs back into a single MS/MMS.
 
-More detailed information about SPW splitting is found [here](/docs/processMeerKAT/config-files#spw-splitting).
+More detailed information about SPW splitting is found [here](https://idia-pipelines.github.io/docs/processMeerKAT/config-files#spw-splitting).
 
 The documentation can be accessed on the [pipelines website](https://idia-pipelines.github.io/docs/processMeerKAT), or on the [Github wiki](https://github.com/idia-astro/pipelines/wiki).

processMeerKAT/aux_scripts/concat.py

@@ -6,6 +6,10 @@
 import glob
 from shutil import copytree
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import config_parser
 from config_parser import validate_args as va
 import bookkeeping

processMeerKAT/config_parser.py

@@ -13,7 +13,7 @@ def parse_args():
     Parse the command line arguments
     """
     parser = argparse.ArgumentParser()
-    parser.add_argument('-C','--config', default=processMeerKAT.CONFIG, required=False, help='Name of the input config file')
+    parser.add_argument('-C','--config', default="myconfig.txt", required=False, help='Name of the input config file')
 
     args, __ = parser.parse_known_args()
 

processMeerKAT/crosscal_scripts/calc_refant.py

@@ -4,13 +4,17 @@
 """
 Calculates the reference antenna
 """
+import os, sys
+import numpy as np
+
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import config_parser
 from config_parser import validate_args as va
 import bookkeeping
 
-import os
-import numpy as np
-
 from casatasks import *
 logfile=casalog.logfile()
 casalog.setlogfile('logs/{SLURM_JOB_NAME}-{SLURM_JOB_ID}.casa'.format(**os.environ))

processMeerKAT/crosscal_scripts/flag_round_1.py

@@ -4,6 +4,10 @@
 import sys
 import os
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import config_parser
 from config_parser import validate_args as va
 import bookkeeping

processMeerKAT/crosscal_scripts/flag_round_2.py

@@ -4,6 +4,11 @@
 import sys
 import os
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
+
 import config_parser
 from config_parser import validate_args as va
 import bookkeeping

processMeerKAT/crosscal_scripts/partition.py

@@ -7,6 +7,11 @@
 import sys
 import os
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
+
 import config_parser
 from config_parser import validate_args as va
 import read_ms
@@ -47,6 +52,13 @@ def main(args,taskvals):
     include_crosshand = va(taskvals, 'run', 'dopol', bool, default=False)
     createmms = va(taskvals, 'crosscal', 'createmms', bool, default=True)
 
+    # HPC Specific Configuration
+    known_hpc_path = os.path.dirname(SCRIPT_DIR)+"/known_hpc.cfg"
+    KNOWN_HPCS, HPC_CONFIG = config_parser.parse_config(known_hpc_path)
+    HPC_NAME = taskvals["run"]["hpc"]
+    HPC_NAME = HPC_NAME if HPC_NAME in KNOWN_HPCS.keys() else "unknown"
+    CPUS_PER_NODE_LIMIT = va(KNOWN_HPCS, HPC_NAME, "CPUS_PER_NODE_LIMIT".lower(), dtype=int)
+
     if nspw > 1:
         casalog.setlogfile('logs/{SLURM_JOB_NAME}-{SLURM_ARRAY_JOB_ID}_{SLURM_ARRAY_TASK_ID}.casa'.format(**os.environ))
     else:
@@ -64,7 +76,7 @@ def main(args,taskvals):
 
     if not include_crosshand and npol == 4:
         npol = 2
-    CPUs = npol if tasks*npol <= processMeerKAT.CPUS_PER_NODE_LIMIT else 1 #hard-code for number of polarisations
+    CPUs = npol if tasks*npol <= CPUS_PER_NODE_LIMIT else 1 #hard-code for number of polarisations
 
     mvis = do_partition(visname, spw, preavg, CPUs, include_crosshand, createmms, spwname)
     mvis = "'{0}'".format(mvis)

processMeerKAT/crosscal_scripts/plot_solutions.py

@@ -1,7 +1,13 @@
 #Copyright (C) 2022 Inter-University Institute for Data Intensive Astronomy
 #See processMeerKAT.py for license details.
 
+import sys
 import os
+
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import config_parser
 from config_parser import validate_args as va
 import bookkeeping

processMeerKAT/crosscal_scripts/plotcal_spw.py

@@ -3,7 +3,13 @@
 
 #!/usr/bin/env python3
 
+import sys
 import os
+
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import glob
 import config_parser
 import traceback

processMeerKAT/crosscal_scripts/quick_tclean.py

@@ -4,6 +4,10 @@
 import sys
 import os
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import config_parser
 from config_parser import validate_args as va
 import bookkeeping

processMeerKAT/crosscal_scripts/setjy.py

@@ -3,6 +3,10 @@
 
 import os, sys, shutil
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import bookkeeping
 from config_parser import validate_args as va
 import numpy as np

processMeerKAT/crosscal_scripts/split.py

@@ -4,6 +4,10 @@
 import sys
 import os
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import config_parser
 import bookkeeping
 from config_parser import validate_args as va

processMeerKAT/crosscal_scripts/xx_yy_apply.py

@@ -4,6 +4,10 @@
 import sys
 import os
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import config_parser
 import bookkeeping
 from config_parser import validate_args as va

processMeerKAT/crosscal_scripts/xx_yy_solve.py

@@ -5,6 +5,10 @@
 import os
 import shutil
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import config_parser
 import bookkeeping
 from config_parser import validate_args as va

processMeerKAT/crosscal_scripts/xy_yx_apply.py

@@ -5,6 +5,10 @@
 import os
 import shutil
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import config_parser
 import bookkeeping, read_ms
 from config_parser import validate_args as va

processMeerKAT/crosscal_scripts/xy_yx_solve.py

@@ -5,6 +5,10 @@
 import os
 import shutil
 
+# Adapt PYTHONPATH to include processMeerKAT
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))
+
 import config_parser
 import bookkeeping
 from config_parser import validate_args as va

processMeerKAT/known_hpc.cfg

@@ -0,0 +1,125 @@
+[DEFAULT]
+    # Set global limits for current ilifu cluster configuration
+    TOTAL_NODES_LIMIT = 79
+    CPUS_PER_NODE_LIMIT = 32
+    NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s
+    MEM_PER_NODE_GB_LIMIT = 232 #237568 MB
+    MEM_PER_NODE_GB_LIMIT_HIGHMEM = 480 #491520 MB
+    ACCOUNTS = ['b03-idia-ag','b05-pipelines-ag']
+
+    # Set global values for paths and file names
+    LOG_DIR = 'logs'
+    CALIB_SCRIPTS_DIR = 'crosscal_scripts'
+    AUX_SCRIPTS_DIR = 'aux_scripts'
+    SELFCAL_SCRIPTS_DIR = 'selfcal_scripts'
+    CONFIG = 'default_config.txt'
+    TMP_CONFIG = '.config.tmp'
+    MASTER_SCRIPT = 'submit_pipeline.sh'
+    SPW_PREFIX = '*:'
+    PARTITION = 'Main'
+    MODULES = ['openmpi/2.1.1']
+    QOS = 'qos-interactive'
+    path_binding = ''
+
+    # Set global values for field, crosscal and SLURM arguments copied to config file, and some of their default values
+    FIELDS_CONFIG_KEYS = [
+        'fluxfield','bpassfield','phasecalfield',
+        'targetfields','extrafields'
+        ]
+    CROSSCAL_CONFIG_KEYS = [
+        'minbaselines','chanbin','width','timeavg','createmms',
+        'keepmms','spw','nspw','calcrefant','refant','standard',
+        'badants','badfreqranges'
+        ]
+    SELFCAL_CONFIG_KEYS = [
+        'nloops','loop','cell','robust','imsize','wprojplanes',
+        'niter','threshold','uvrange','nterms','gridder','deconvolver',
+        'solint','calmode','discard_nloops','gaintype',
+        'outlier_threshold','flag'
+        ]
+    IMAGING_CONFIG_KEYS = [
+        'cell', 'robust', 'imsize', 'wprojplanes', 'niter',
+        'threshold', 'multiscale', 'nterms', 'gridder',
+        'deconvolver', 'restoringbeam', 'specmode',
+        'stokes', 'mask', 'rmsmap'
+        ]
+    SLURM_CONFIG_STR_KEYS = [
+        'container','mpi_wrapper','partition','time','name',
+        'dependencies','exclude','account','reservation'
+        ]
+    SLURM_CONFIG_KEYS_BASE = [
+        'nodes','ntasks_per_node','mem','plane','submit',
+        'precal_scripts','postcal_scripts','scripts',
+        'verbose','modules'
+        ]
+    CONTAINER = "/idia/software/containers/casa-6.3.simg"
+    MPI_WRAPPER = 'mpirun'
+    PRECAL_SCRIPTS = [('calc_refant.py',False,''),('partition.py',True,'')] #Scripts run before calibration at top level directory when nspw > 1
+    POSTCAL_SCRIPTS = [
+        ('concat.py',False,''),
+        ('plotcal_spw.py', False, ''),
+        ('selfcal_part1.py',True,''),
+        ('selfcal_part2.py',False,''),
+        ('science_image.py', True, '')
+        ] #Scripts run after calibration at top level directory when nspw > 1
+    SCRIPTS = [
+        ('validate_input.py',False,''),
+        ('flag_round_1.py',True,''),
+        ('calc_refant.py',False,''),
+        ('setjy.py',True,''),
+        ('xx_yy_solve.py',False,''),
+        ('xx_yy_apply.py',True,''),
+        ('flag_round_2.py',True,''),
+        ('xx_yy_solve.py',False,''),
+        ('xx_yy_apply.py',True,''),
+        ('split.py',True,''),
+        ('quick_tclean.py',True,'')
+        ]
+    # sbatch_file_base must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them.
+    submission_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --account={account}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --partition={partition}\n#SBATCH --time={time}"
+
+[ilifu]
+    # [DEFAULT] section defined at the top of this file represents the ilifu configuration.
+
+[unknown]
+    # Differences to default: memory / node limits are functionally unlimited.
+    TOTAL_NODES_LIMIT = 5096
+    CPUS_PER_NODE_LIMIT = 1024
+    NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s
+    MEM_PER_NODE_GB_LIMIT = 5000 #237568 MB
+    MEM_PER_NODE_GB_LIMIT_HIGHMEM = 5000 #491520 MB
+    ACCOUNTS = ['']
+
+[galahad]
+    # Specify differences to ilifu
+    TOTAL_NODES_LIMIT = 17 # Total number of hmem nodes
+    CPUS_PER_NODE_LIMIT = 16 # Number of threads on hmem nodes
+    NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s
+    MEM_PER_NODE_GB_LIMIT = 1500 # GB; 1.5TB available
+    MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1500 # GB; 1.5 TB availble
+    ACCOUNTS = [''] # List of allowed accounts; Not currently used: see sbatch_file_base ### update me!!!
+    CONTAINER = '/share/nas/mbowles/casa-6_v2.simg' # Previously used '/share/nas/mbowles/mightee/casa-stable.simg'
+    PARTITION = 'CLUSTER'
+    QOS = 'Normal'
+    MPI_WRAPPER = 'mpirun --mca oob tcp'
+    MODULES = ['openmpi-2.1.1']
+    path_binding = '--bind /share:/share '
+    # Must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them.
+    submission_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --time={time}"
+
+[petrichor]
+    # Specify differences to ilifu
+    TOTAL_NODES_LIMIT = 110
+    CPUS_PER_NODE_LIMIT = 64
+    NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s
+    MEM_PER_NODE_GB_LIMIT = 512 # GB
+    MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1000 #
+    ACCOUNTS = [''] # List of allowed accounts; Not currently used: see sbatch_file_base ### update me!!!
+    CONTAINER = '/scratch1/tho822/containers/casa-pipeline/casa-6.1.2.7-modular.simg' #
+    PARTITION = 'defq'
+    QOS = 'express'
+    MPI_WRAPPER = 'mpirun'
+    MODULES = ['singularity', 'use.own', 'openmpi/2.1.1']
+    path_binding = '--bind /scratch1:/scratch1,/scratch2:/scratch2 '
+    # Must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them.
+    submission_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --time={time}"