forked from rchudasa/BsMMGAnalysis
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adding the multi crab submission script
Usage ``` python multiCrab_ParkingD.py --crabCmd submit python multiCrab_ParkingD.py --crabCmd status --workArea WorkArea ```
- Loading branch information
Showing
1 changed file
with
133 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,133 @@ | ||
""" | ||
This is a small script that does the equivalent of multicrab. | ||
""" | ||
import os | ||
from optparse import OptionParser | ||
|
||
import CRABClient | ||
from CRABAPI.RawCommand import crabCommand | ||
from CRABClient.ClientExceptions import ClientException | ||
from httplib import HTTPException | ||
|
||
|
||
def getOptions(): | ||
""" | ||
Parse and return the arguments provided by the user. | ||
""" | ||
usage = ("Usage: %prog --crabCmd CMD [--workArea WAD --crabCmdOpts OPTS]" | ||
"\nThe multicrab command executes 'crab CMD OPTS' for each project directory contained in WAD" | ||
"\nUse multicrab -h for help") | ||
|
||
parser = OptionParser(usage=usage) | ||
|
||
parser.add_option('-c', '--crabCmd', | ||
dest = 'crabCmd', | ||
default = '', | ||
help = "crab command", | ||
metavar = 'CMD') | ||
|
||
parser.add_option('-w', '--workArea', | ||
dest = 'workArea', | ||
default = '', | ||
help = "work area directory (only if CMD != 'submit')", | ||
metavar = 'WAD') | ||
|
||
parser.add_option('-o', '--crabCmdOpts', | ||
dest = 'crabCmdOpts', | ||
default = '', | ||
help = "options for crab command CMD", | ||
metavar = 'OPTS') | ||
|
||
(options, arguments) = parser.parse_args() | ||
|
||
if arguments: | ||
parser.error("Found positional argument(s): %s." % (arguments)) | ||
if not options.crabCmd: | ||
parser.error("(-c CMD, --crabCmd=CMD) option not provided.") | ||
if options.crabCmd != 'submit': | ||
if not options.workArea: | ||
parser.error("(-w WAR, --workArea=WAR) option not provided.") | ||
if not os.path.isdir(options.workArea): | ||
parser.error("'%s' is not a valid directory." % (options.workArea)) | ||
|
||
return options | ||
|
||
|
||
def main(): | ||
|
||
options = getOptions() | ||
|
||
# The submit command needs special treatment. | ||
if options.crabCmd == 'submit': | ||
|
||
#-------------------------------------------------------- | ||
# This is the base config: | ||
#-------------------------------------------------------- | ||
from CRABClient.UserUtilities import config | ||
config = config() | ||
|
||
config.General.requestName = None | ||
config.General.workArea = 'WorkArea' | ||
|
||
config.JobType.pluginName = 'Analysis' | ||
config.JobType.psetName = 'data2018_cfg.py' | ||
|
||
config.Data.inputDataset = None | ||
config.Data.splitting = 'FileBased' | ||
config.Data.unitsPerJob = 20 | ||
config.Data.totalUnits = -1 | ||
config.Data.outputDatasetTag = None | ||
|
||
config.Data.outLFNDirBase = '/store/user/athachay/BsToMuMuGamma/BParkingStudies/data2018Ntuples' | ||
config.Site.storageSite = 'T2_IN_TIFR' # Choose your site. | ||
#-------------------------------------------------------- | ||
|
||
# Will submit one task for each of these input datasets. | ||
inputDatasets = [ | ||
'/ParkingBPH1/Run2018D-20Jun2021_UL2018-v1/AOD', | ||
'/ParkingBPH3/Run2018D-20Jun2021_UL2018-v1/AOD', | ||
'/ParkingBPH4/Run2018D-20Jun2021_UL2018-v1/AOD', | ||
'/ParkingBPH5/Run2018D-20Jun2021_UL2018-v1/AOD' | ||
] | ||
|
||
for inDS in inputDatasets: | ||
# inDS is of the form /A/B/C. Since B is unique for each inDS, use this in the CRAB request name. | ||
config.General.requestName = inDS.split('/')[1]+'_'+inDS.split('/')[2] | ||
if 'ParkingBPH1' in config.General.requestName: | ||
config.General.requestName=inDS.split('/')[2] | ||
config.Data.inputDataset = inDS | ||
config.Data.outputDatasetTag = '%s_%s' % (config.General.workArea, config.General.requestName) | ||
# Submit. | ||
try: | ||
print "Submitting for input dataset %s" % (inDS) | ||
#print "crabCommand(",options.crabCmd,", config = ,",config,", ",options.crabCmdOpts.split(),")" | ||
crabCommand(options.crabCmd, config = config, *options.crabCmdOpts.split()) | ||
except HTTPException as hte: | ||
print "Submission for input dataset %s failed: %s" % (inDS, hte.headers) | ||
except ClientException as cle: | ||
print "Submission for input dataset %s failed: %s" % (inDS, cle) | ||
|
||
# All other commands can be simply executed. | ||
elif options.workArea: | ||
|
||
for dir in os.listdir(options.workArea): | ||
projDir = os.path.join(options.workArea, dir) | ||
if not os.path.isdir(projDir): | ||
continue | ||
# Execute the crab command. | ||
msg = "Executing (the equivalent of): crab %s --dir %s %s" % (options.crabCmd, projDir, options.crabCmdOpts) | ||
print "-"*len(msg) | ||
print msg | ||
print "-"*len(msg) | ||
try: | ||
#print "crabCommand(",options.crabCmd," dir = ",projDir,", ",options.crabCmdOpts.split() , ")" | ||
crabCommand(options.crabCmd, dir = projDir, *options.crabCmdOpts.split()) | ||
except HTTPException as hte: | ||
print "Failed executing command %s for task %s: %s" % (options.crabCmd, projDir, hte.headers) | ||
except ClientException as cle: | ||
print "Failed executing command %s for task %s: %s" % (options.crabCmd, projDir, cle) | ||
|
||
|
||
if __name__ == '__main__': | ||
main() | ||
|