CMS 3D CMS Logo
runTheMatrix.py
Go to the documentation of this file.
1 #!/usr/bin/env python3
2 from __future__ import print_function
3 import sys, os
4 
5 from Configuration.PyReleaseValidation.MatrixReader import MatrixReader
6 from Configuration.PyReleaseValidation.MatrixRunner import MatrixRunner
7 from Configuration.PyReleaseValidation.MatrixInjector import MatrixInjector,performInjectionOptionTest
8 
9 # ================================================================================
10 
11 def showRaw(opt):
12 
13  mrd = MatrixReader(opt)
14  mrd.showRaw(opt.useInput, opt.refRel, opt.fromScratch, opt.raw, opt.step1Only, selected=opt.testList)
15 
16  return 0
17 
18 # ================================================================================
19 
20 def runSelected(opt):
21 
22  mrd = MatrixReader(opt)
23  mrd.prepare(opt.useInput, opt.refRel, opt.fromScratch)
24 
25  # test for wrong input workflows
26  if opt.testList:
27  definedWf = [dwf.numId for dwf in mrd.workFlows]
28  definedSet = set(definedWf)
29  testSet = set(opt.testList)
30  undefSet = testSet - definedSet
31  if len(undefSet)>0: raise ValueError('Undefined workflows: '+', '.join(map(str,list(undefSet))))
32  duplicates = [wf for wf in testSet if definedWf.count(wf)>1 ]
33  if len(duplicates)>0: raise ValueError('Duplicated workflows: '+', '.join(map(str,list(duplicates))))
34 
35  ret = 0
36  if opt.show:
37  mrd.show(opt.testList, opt.extended, opt.cafVeto)
38  if opt.testList : print('selected items:', opt.testList)
39  else:
40  mRunnerHi = MatrixRunner(mrd.workFlows, opt.nProcs, opt.nThreads)
41  ret = mRunnerHi.runTests(opt)
42 
43  if opt.wmcontrol:
44  if ret!=0:
45  print('Cannot go on with wmagent injection with failing workflows')
46  else:
47  wfInjector = MatrixInjector(opt,mode=opt.wmcontrol,options=opt.wmoptions)
48  ret= wfInjector.prepare(mrd,
49  mRunnerHi.runDirs)
50  if ret==0:
51  wfInjector.upload()
52  wfInjector.submit()
53  return ret
54 
55 # ================================================================================
56 
57 if __name__ == '__main__':
58 
59  #this can get out of here
60  predefinedSet={
61  'limited' : [
62  # See README for further details
63 
66  5.1, # TTbar_8TeV_TuneCUETP8M1 FastSim
67  8, # RelValBeamHalo Cosmics
68  9.0, # RelValHiggs200ChargedTaus
69  25, # RelValTTbar
70  101.0, # SingleElectronE120EHCAL + ECALHCAL.customise + fullMixCustomize_cff.setCrossingFrameOn
71 
72  # Run2
73  7.3, # UndergroundCosmicSPLooseMu
74  1306.0, # RelValSingleMuPt1_UP15
75  1330, # RelValZMM_13
76  135.4, # ZEE_13TeV_TuneCUETP8M1
77  25202.0, # RelValTTbar_13 PU = AVE_35_BX_25ns
78  250202.181, # RelValTTbar_13 PREMIX
79 
80  # Run3
81  11634.0, # TTbar_14TeV 2021
82  13234.0, # RelValTTbar_14TeV 2021 FastsSim
83  12434.0, # RelValTTbar_14TeV 2023
84  12834.0, # RelValTTbar_14TeV 2024
85  12846.0, # RelValZEE_13 2024
86  13034.0, # RelValTTbar_14TeV 2024 PU = Run3_Flat55To75_PoissonOOTPU
87  12834.7, # RelValTTbar_14TeV 2024 mkFit
88  16834.0, # RelValTTbar_14TeV 2025
89  14034.0, # RelValTTbar_14TeV Run3_2023_FastSim
90  14234.0, # RelValTTbar_14TeV Run3_2023_FastSim PU = Run3_Flat55To75_PoissonOOTPU
91  2500.201, # RelValTTbar_14TeV NanoAOD from existing MINI
92 
93  # Phase2
94  29634.0, # RelValTTbar_14TeV phase2_realistic_T33 ExtendedRun4D110 (Phase-2 baseline)
95  24834.911, # Previous DD4hep baseline for monitoring the stability of DD4hep workflow
96  29634.911, # TTbar_14TeV_TuneCP5 phase2_realistic_T33 DD4hepExtendedRun4D110 DD4Hep (HLLHC14TeV BeamSpot)
97  29834.999, # RelValTTbar_14TeV (PREMIX) phase2_realistic_T33 ExtendedRun4D110 AVE_50_BX_25ns_m3p3
98  29696.0, # RelValCloseByPGun_CE_E_Front_120um phase2_realistic_T33 ExtendedRun4D110
99  29700.0, # RelValCloseByPGun_CE_H_Coarse_Scint phase2_realistic_T33 ExtendedRun4D110
100  #23234.0, # Need new workflow with HFNose
101  29634.75, # RelValTTbar_14TeV phase2_realistic_T33 ExtendedRun4D110 (Phase-2 baseline - but using timing menu, and only up to step 2)
102 
103 
105  4.22, # Run2011A Cosmics
106  4.53, # Run2012B Photon miniAODs
107  1000, # Run2011A MinimumBias Prompt RecoTLR.customisePrompt
108  1001, # Run2011A MinimumBias Data+Express
109 
110  136.731, # Run2016B SinglePhoton
111  136.793, # Run2017C DoubleEG
112  136.874, # Run2018C EGamma
113 
114 
116  139.001, # Run2021 MinimumBias Commissioning2021
117 
118  # 2022
119  2022.002001, # Run2022C ZeroBias
120  2022.000001, # Run2022C JetHT
121 
122  # 2023
123  2023.002001, # Run2023D ZeroBias
124  2023.002001, # Run2023D MuonEG
125 
126  # 2024
127  2024.000001, # Run2024B ZeroBias
128  2024.101001, # Run2024C BTagMu
129  2024.202001, # Run2024D JetMET0
130  2024.303001, # Run2024E DisplacedJet
131 
132 
135  140.56, # HIRun2018A HIHardProbes Run2_2018_pp_on_AA
136 
137  312.0, # Pyquen_ZeemumuJets_pt10_2760GeV PU : HiMixGEN
138 
139  ],
140  'jetmc': [5.1, 13, 15, 25, 38, 39], #MC
141  'metmc' : [5.1, 15, 25, 37, 38, 39], #MC
142  'muonmc' : [5.1, 124.4, 124.5, 20, 21, 22, 23, 25, 30], #MC
143  }
144 
145 
146  import argparse
147  usage = 'usage: runTheMatrix.py --show -s '
148 
149  parser = argparse.ArgumentParser(usage,formatter_class=argparse.ArgumentDefaultsHelpFormatter)
150 
151  parser.add_argument('-b','--batchName',
152  help='relval batch: suffix to be appended to Campaign name',
153  dest='batchName',
154  default='')
155 
156  parser.add_argument('-m','--memoryOffset',
157  help='memory of the wf for single core',
158  dest='memoryOffset',
159  type=int,
160  default=3000)
161 
162  parser.add_argument('--addMemPerCore',
163  help='increase of memory per each n > 1 core: memory(n_core) = memoryOffset + (n_core-1) * memPerCore',
164  dest='memPerCore',
165  type=int,
166  default=1500)
167 
168  parser.add_argument('-j','--nproc',
169  help='number of processes. 0 Will use 4 processes, not execute anything but create the wfs',
170  dest='nProcs',
171  type=int,
172  default=4)
173 
174  parser.add_argument('-t','--nThreads',
175  help='number of threads per process to use in cmsRun.',
176  dest='nThreads',
177  type=int,
178  default=1)
179 
180  parser.add_argument('--nStreams',
181  help='number of streams to use in cmsRun.',
182  dest='nStreams',
183  type=int,
184  default=0)
185 
186  parser.add_argument('--nEvents',
187  help='number of events to process in cmsRun. If 0 will use the standard 10 events.',
188  dest='nEvents',
189  type=int,
190  default=0)
191 
192  parser.add_argument('--numberEventsInLuminosityBlock',
193  help='number of events in a luminosity block',
194  dest='numberEventsInLuminosityBlock',
195  type=int,
196  default=-1)
197 
198  parser.add_argument('-n','--showMatrix',
199  help='Only show the worflows. Use --ext to show more',
200  dest='show',
201  default=False,
202  action='store_true')
203 
204  parser.add_argument('-e','--extended',
205  help='Show details of workflows, used with --show',
206  dest='extended',
207  default=False,
208  action='store_true')
209 
210  parser.add_argument('-s','--selected',
211  help='Run a pre-defined selected matrix of wf. Deprecated, please use -l limited',
212  dest='restricted',
213  default=False,
214  action='store_true')
215 
216  parser.add_argument('-l','--list',
217  help='Comma separated list of workflow to be shown or ran. Possible keys are also '+str(predefinedSet.keys())+'. and wild card like muon, or mc',
218  dest='testList',
219  default=None)
220 
221  parser.add_argument('-f','--failed-from',
222  help='Provide a matrix report to specify the workflows to be run again. Augments the -l option if specified already',
223  dest='failed_from',
224  default=None)
225 
226  parser.add_argument('-r','--raw',
227  help='Temporary dump the .txt needed for prodAgent interface. To be discontinued soon. Argument must be the name of the set (standard, pileup,...)',
228  dest='raw')
229 
230  parser.add_argument('-i','--useInput',
231  help='Use recyling where available. Either all, or a comma separated list of wf number.',
232  dest='useInput',
233  type=lambda x: x.split(','),
234  default=None)
235 
236  parser.add_argument('-w','--what',
237  help='Specify the set to be used. Argument must be the name of a set (standard, pileup,...) or multiple sets separated by commas (--what standard,pileup )',
238  dest='what',
239  default='all')
240 
241  parser.add_argument('--step1',
242  help='Used with --raw. Limit the production to step1',
243  dest='step1Only',
244  default=False)
245 
246  parser.add_argument('--maxSteps',
247  help='Only run maximum on maxSteps. Used when we are only interested in first n steps.',
248  dest='maxSteps',
249  default=9999,
250  type=int)
251 
252  parser.add_argument('--fromScratch',
253  help='Comma separated list of wf to be run without recycling. all is not supported as default.',
254  dest='fromScratch',
255  type=lambda x: x.split(','),
256  default=None)
257 
258  parser.add_argument('--refRelease',
259  help='Allow to modify the recycling dataset version',
260  dest='refRel',
261  default=None)
262 
263  parser.add_argument('--wmcontrol',
264  help='Create the workflows for injection to WMAgent. In the WORKING. -wmcontrol init will create the the workflows, -wmcontrol test will dryRun a test, -wmcontrol submit will submit to wmagent',
265  choices=['init','test','submit','force'],
266  dest='wmcontrol',
267  default=None)
268 
269  parser.add_argument('--revertDqmio',
270  help='When submitting workflows to wmcontrol, force DQM outout to use pool and not DQMIO',
271  choices=['yes','no'],
272  dest='revertDqmio',
273  default='no')
274 
275  parser.add_argument('--optionswm',
276  help='Specify a few things for wm injection',
277  default='',
278  dest='wmoptions')
279 
280  parser.add_argument('--keep',
281  help='allow to specify for which comma separated steps the output is needed',
282  default=None)
283 
284  parser.add_argument('--label',
285  help='allow to give a special label to the output dataset name',
286  default='')
287 
288  parser.add_argument('--command',
289  help='provide a way to add additional command to all of the cmsDriver commands in the matrix',
290  dest='command',
291  action='append',
292  default=None)
293 
294  parser.add_argument('--apply',
295  help='allow to use the --command only for 1 comma separeated',
296  dest='apply',
297  default=None)
298 
299  parser.add_argument('--workflow',
300  help='define a workflow to be created or altered from the matrix',
301  action='append',
302  dest='workflow',
303  default=None)
304 
305  parser.add_argument('--dryRun',
306  help='do not run the wf at all',
307  action='store_true',
308  dest='dryRun',
309  default=False)
310 
311  parser.add_argument('--testbed',
312  help='workflow injection to cmswebtest (you need dedicated rqmgr account)',
313  dest='testbed',
314  default=False,
315  action='store_true')
316 
317  parser.add_argument('--noCafVeto',
318  help='Run from any source, ignoring the CAF label',
319  dest='cafVeto',
320  default=True,
321  action='store_false')
322 
323  parser.add_argument('--overWrite',
324  help='Change the content of a step for another. List of pairs.',
325  dest='overWrite',
326  default=None)
327 
328  parser.add_argument('--noRun',
329  help='Remove all run list selection from wfs',
330  dest='noRun',
331  default=False,
332  action='store_true')
333 
334  parser.add_argument('--das-options',
335  help='Options to be passed to dasgoclient.',
336  dest='dasOptions',
337  default="--limit 0",
338  action='store')
339 
340  parser.add_argument('--job-reports',
341  help='Dump framework job reports',
342  dest='jobReports',
343  default=False,
344  action='store_true')
345 
346  parser.add_argument('--ibeos',
347  help='Use IB EOS site configuration',
348  dest='IBEos',
349  default=False,
350  action='store_true')
351 
352  parser.add_argument('--sites',
353  help='Run DAS query to get data from a specific site. Set it to empty string to search all sites.',
354  dest='dasSites',
355  default='T2_CH_CERN',
356  action='store')
357 
358  parser.add_argument('--interactive',
359  help="Open the Matrix interactive shell",
360  action='store_true',
361  default=False)
362 
363  parser.add_argument('--dbs-url',
364  help='Overwrite DbsUrl value in JSON submitted to ReqMgr2',
365  dest='dbsUrl',
366  default=None,
367  action='store')
368 
369  gpugroup = parser.add_argument_group('GPU-related options','These options are only meaningful when --gpu is used, and is not set to forbidden.')
370 
371  gpugroup.add_argument('--gpu','--requires-gpu',
372  help='Enable GPU workflows. Possible options are "forbidden" (default), "required" (implied if no argument is given), or "optional".',
373  dest='gpu',
374  choices=['forbidden', 'optional', 'required'],
375  nargs='?',
376  const='required',
377  default='forbidden',
378  action='store')
379 
380  gpugroup.add_argument('--gpu-memory',
381  help='Specify the minimum amount of GPU memory required by the job, in MB.',
382  dest='GPUMemoryMB',
383  type=int,
384  default=8000)
385 
386  gpugroup.add_argument('--cuda-capabilities',
387  help='Specify a comma-separated list of CUDA "compute capabilities", or GPU hardware architectures, that the job can use.',
388  dest='CUDACapabilities',
389  type=lambda x: x.split(','),
390  default='6.0,6.1,6.2,7.0,7.2,7.5,8.0,8.6')
391 
392  # read the CUDA runtime version included in CMSSW
393  cudart_version = None
394  libcudart = os.path.realpath(os.path.expandvars('$CMSSW_RELEASE_BASE/external/$SCRAM_ARCH/lib/libcudart.so'))
395  if os.path.isfile(libcudart):
396  cudart_basename = os.path.basename(libcudart)
397  cudart_version = '.'.join(cudart_basename.split('.')[2:4])
398  gpugroup.add_argument('--cuda-runtime',
399  help='Specify major and minor version of the CUDA runtime used to build the application.',
400  dest='CUDARuntime',
401  default=cudart_version)
402 
403  gpugroup.add_argument('--force-gpu-name',
404  help='Request a specific GPU model, e.g. "Tesla T4" or "NVIDIA GeForce RTX 2080". The default behaviour is to accept any supported GPU.',
405  dest='GPUName',
406  default='')
407 
408  gpugroup.add_argument('--force-cuda-driver-version',
409  help='Request a specific CUDA driver version, e.g. 470.57.02. The default behaviour is to accept any supported CUDA driver version.',
410  dest='CUDADriverVersion',
411  default='')
412 
413  gpugroup.add_argument('--force-cuda-runtime-version',
414  help='Request a specific CUDA runtime version, e.g. 11.4. The default behaviour is to accept any supported CUDA runtime version.',
415  dest='CUDARuntimeVersion',
416  default='')
417 
418  opt = parser.parse_args()
419  if opt.command: opt.command = ' '.join(opt.command)
420  os.environ["CMSSW_DAS_QUERY_SITES"]=opt.dasSites
421  if opt.failed_from:
422  rerunthese=[]
423  with open(opt.failed_from,'r') as report:
424  for report_line in report:
425  if 'FAILED' in report_line:
426  to_run,_=report_line.split('_',1)
427  rerunthese.append(to_run)
428  if opt.testList:
429  opt.testList+=','.join(['']+rerunthese)
430  else:
431  opt.testList = ','.join(rerunthese)
432 
433  if opt.IBEos:
434  os.environ["CMSSW_USE_IBEOS"]="true"
435  if opt.restricted:
436  print('Deprecated, please use -l limited')
437  if opt.testList: opt.testList+=',limited'
438  else: opt.testList='limited'
439 
440  def stepOrIndex(s):
441  if s.isdigit():
442  return int(s)
443  else:
444  return s
445  if opt.apply:
446  opt.apply=map(stepOrIndex,opt.apply.split(','))
447  if opt.keep:
448  opt.keep=map(stepOrIndex,opt.keep.split(','))
449 
450  if opt.testList:
451  testList=[]
452  for entry in opt.testList.split(','):
453  if not entry: continue
454  mapped=False
455  for k in predefinedSet:
456  if k.lower().startswith(entry.lower()) or k.lower().endswith(entry.lower()):
457  testList.extend(predefinedSet[k])
458  mapped=True
459  break
460  if not mapped:
461  try:
462  testList.append(float(entry))
463  except:
464  print(entry,'is not a possible selected entry')
465 
466  opt.testList = list(set(testList))
467 
468  if opt.wmcontrol:
469  performInjectionOptionTest(opt)
470  if opt.overWrite:
471  opt.overWrite=eval(opt.overWrite)
472  if opt.interactive:
473  import cmd
474  from colorama import Fore, Style
475  from os import isatty
476  import subprocess
477  import time
478 
479  class TheMatrix(cmd.Cmd):
480  intro = "Welcome to the Matrix (? for help)"
481  prompt = "matrix> "
482 
483  def __init__(self, opt):
484  cmd.Cmd.__init__(self)
485  self.opt_ = opt
486  self.matrices_ = {}
487  tmp = MatrixReader(self.opt_)
488  self.processes_ = dict()
489  for what in tmp.files:
490  what = what.replace('relval_','')
491  self.opt_.what = what
492  self.matrices_[what] = MatrixReader(self.opt_)
493  self.matrices_[what].prepare(self.opt_.useInput, self.opt_.refRel,
494  self.opt_.fromScratch)
495  os.system("clear")
496 
497  def do_clear(self, arg):
498  """Clear the screen, put prompt at the top"""
499  os.system("clear")
500 
501  def do_exit(self, arg):
502  print("Leaving the Matrix")
503  return True
504 
505  def default(self, inp):
506  if inp == 'x' or inp == 'q':
507  return self.do_exit(inp)
508  else:
509  is_pipe = not isatty(sys.stdin.fileno())
510  print(Fore.RED + "Error: " + Fore.RESET + "unrecognized command.")
511  # Quit only if given a piped command.
512  if is_pipe:
513  sys.exit(1)
514 
515  def help_predefined(self):
516  print("\n".join(["predefined [predef1 [...]]\n",
517  "Run w/o argument, it will print the list of known predefined workflows.",
518  "Run with space-separated predefined workflows, it will print the workflow-ids registered to them"]))
519 
520  def complete_predefined(self, text, line, start_idx, end_idx):
521  if text and len(text) > 0:
522  return [t for t in predefinedSet.keys() if t.startswith(text)]
523  else:
524  return predefinedSet.keys()
525 
526  def do_predefined(self, arg):
527  """Print the list of predefined workflows"""
528  print("List of predefined workflows")
529  if arg:
530  for w in arg.split():
531  if w in predefinedSet.keys():
532  print("Predefined Set: %s" % w)
533  print(predefinedSet[w])
534  else:
535  print("Unknown Set: %s" % w)
536  else:
537  print("[ " + Fore.RED + ", ".join([str(k) for k in predefinedSet.keys()]) + Fore.RESET + " ]")
538 
539  def help_showWorkflow(self):
540  print("\n".join(["showWorkflow [workflow1 [...]]\n",
541  "Run w/o arguments, it will print the list of registered macro-workflows.",
542  "Run with space-separated workflows, it will print the full list of workflow-ids registered to them"]))
543 
544  def complete_showWorkflow(self, text, line, start_idx, end_idx):
545  if text and len(text) > 0:
546  return [t for t in self.matrices_.keys() if t.startswith(text)]
547  else:
548  return self.matrices_.keys()
549 
550  def do_showWorkflow(self, arg):
551  if arg == '':
552  print("Available workflows:")
553  for k in self.matrices_.keys():
554  print(Fore.RED + Style.BRIGHT + k)
555  print(Style.RESET_ALL)
556  else:
557  selected = arg.split()
558  for k in selected:
559  if k not in self.matrices_.keys():
560  print("Unknown workflow %s: skipping" % k)
561  else:
562  for wfl in self.matrices_[k].workFlows:
563  print("%s %s" % (Fore.BLUE + str(wfl.numId) + Fore.RESET,
564  Fore.GREEN + wfl.nameId + Fore.RESET))
565  print("%s contains %d workflows" % (Fore.RED + k + Fore.RESET, len(self.matrices_[k].workFlows)))
566 
567  def do_runWorkflow(self, arg):
568  # Split the input arguments into a list
569  args = arg.split()
570  if len(args) < 2:
571  print(Fore.RED + Style.BRIGHT + "Wrong number of parameters passed")
572  print(Style.RESET_ALL)
573  return
574  workflow_class = args[0]
575  workflow_id = args[1]
576  passed_down_args = list()
577  if len(args) > 2:
578  passed_down_args = args[2:]
579  print(Fore.YELLOW + Style.BRIGHT + "Running with the following options:\n")
580  print(Fore.GREEN + Style.BRIGHT + "Workflow class: {}".format(workflow_class))
581  print(Fore.GREEN + Style.BRIGHT + "Workflow ID: {}".format(workflow_id))
582  print(Fore.GREEN + Style.BRIGHT + "Additional runTheMatrix options: {}".format(passed_down_args))
583  print(Style.RESET_ALL)
584  if workflow_class not in self.matrices_.keys():
585  print(Fore.RED + Style.BRIGHT + "Unknown workflow selected: {}".format(workflow_class))
586  print("Available workflows:")
587  for k in self.matrices_.keys():
588  print(Fore.RED + Style.BRIGHT + k)
589  print(Style.RESET_ALL)
590  return
591  wflnums = [x.numId for x in self.matrices_[workflow_class].workFlows]
592  if float(workflow_id) not in wflnums:
593  print(Fore.RED + Style.BRIGHT + "Unknown workflow {}".format(workflow_id))
594  print(Fore.GREEN + Style.BRIGHT)
595  print(wflnums)
596  print(Style.RESET_ALL)
597  return
598  if workflow_id in self.processes_.keys():
599  # Check if the process is still active
600  if self.processes_[workflow_id][0].poll() is None:
601  print(Fore.RED + Style.BRIGHT + "Workflow {} already running!".format(workflow_id))
602  print(Style.RESET_ALL)
603  return
604  # If it was there but it's gone, proceeed and update the value for the same key
605  # run a job, redirecting standard output and error to files
606  lognames = ['stdout', 'stderr']
607  logfiles = tuple('%s_%s_%s.log' % (workflow_class, workflow_id, name) for name in lognames)
608  stdout = open(logfiles[0], 'w')
609  stderr = open(logfiles[1], 'w')
610  command = ('runTheMatrix.py', '-w', workflow_class, '-l', workflow_id)
611  if len(passed_down_args) > 0:
612  command += tuple(passed_down_args)
613  print(command)
614  p = subprocess.Popen(command,
615  stdout = stdout,
616  stderr = stderr)
617  self.processes_[workflow_id] = (p, time.time())
618 
619 
620  def complete_runWorkflow(self, text, line, start_idx, end_idx):
621  if text and len(text) > 0:
622  return [t for t in self.matrices_.keys() if t.startswith(text)]
623  else:
624  return self.matrices_.keys()
625 
626  def help_runWorkflow(self):
627  print("\n".join(["runWorkflow workflow_class workflow_id\n",
628  "This command will launch a new and independent process that invokes",
629  "the command:\n",
630  "runTheMatrix.py -w workflow_class -l workflow_id [runTheMatrix.py options]",
631  "\nYou can specify just one workflow_class and workflow_id per invocation.",
632  "The job will continue even after quitting the interactive session.",
633  "stdout and stderr of the new process will be automatically",
634  "redirected to 2 logfiles whose names contain the workflow_class",
635  "and workflow_id. Mutiple command can be issued one after the other.",
636  "The working directory of the new process will be the directory",
637  "from which the interactive session has started.",
638  "Autocompletion is available for workflow_class, but",
639  "not for workflow_id. Supplying a wrong workflow_class or",
640  "a non-existing workflow_id for a valid workflow_class",
641  "will trigger an error and no process will be invoked.",
642  "The interactive shell will keep track of all active processes",
643  "and will prevent the accidental resubmission of an already",
644  "active jobs."]))
645 
646  def do_jobs(self, args):
647  print(Fore.GREEN + Style.BRIGHT + "List of jobs:")
648  for w in self.processes_.keys():
649  if self.processes_[w][0].poll() is None:
650  print(Fore.YELLOW + Style.BRIGHT + "Active job: {} since {:.2f} seconds.".format(w, time.time() - self.processes_[w][1]))
651  else:
652  print(Fore.RED + Style.BRIGHT + "Done job: {}".format(w))
653  print(Style.RESET_ALL)
654 
655  def help_jobs(self):
656  print("\n".join(["Print a full list of active and done jobs submitted",
657  "in the ongoing interactive session"]))
658 
659  def help_searchInWorkflow(self):
660  print("\n".join(["searchInWorkflow wfl_name search_regexp\n",
661  "This command will search for a match within all workflows registered to wfl_name.",
662  "The search is done on both the workflow name and the names of steps registered to it."]))
663 
664  def complete_searchInWorkflow(self, text, line, start_idx, end_idx):
665  if text and len(text) > 0:
666  return [t for t in self.matrices_.keys() if t.startswith(text)]
667  else:
668  return self.matrices_.keys()
669 
670  def do_searchInWorkflow(self, arg):
671  args = arg.split()
672  if len(args) < 2:
673  print("searchInWorkflow name regexp")
674  return
675  if args[0] not in self.matrices_.keys():
676  print("Unknown workflow")
677  return
678  import re
679  pattern = None
680  try:
681  pattern = re.compile(args[1])
682  except:
683  print("Failed to compile regexp %s" % args[1])
684  return
685  counter = 0
686  for wfl in self.matrices_[args[0]].workFlows:
687  if re.match(pattern, wfl.nameId):
688  print("%s %s" % (Fore.BLUE + str(wfl.numId) + Fore.RESET,
689  Fore.GREEN + wfl.nameId + Fore.RESET))
690  counter +=1
691  print("Found %s compatible workflows inside %s" % (Fore.RED + str(counter) + Fore.RESET,
692  Fore.YELLOW + str(args[0])) + Fore.RESET)
693 
694  def help_search(self):
695  print("\n".join(["search search_regexp\n",
696  "This command will search for a match within all workflows registered.",
697  "The search is done on both the workflow name and the names of steps registered to it."]))
698 
699  def do_search(self, arg):
700  args = arg.split()
701  if len(args) < 1:
702  print("search regexp")
703  return
704  for wfl in self.matrices_.keys():
705  self.do_searchInWorkflow(' '.join([wfl, args[0]]))
706 
707  def help_dumpWorkflowId(self):
708  print("\n".join(["dumpWorkflowId [wfl-id1 [...]]\n",
709  "Dumps the details (cmsDriver commands for all steps) of the space-separated workflow-ids in input."]))
710 
711  def do_dumpWorkflowId(self, arg):
712  wflids = arg.split()
713  if len(wflids) == 0:
714  print("dumpWorkflowId [wfl-id1 [...]]")
715  return
716 
717  fmt = "[%s]: %s\n"
718  maxLen = 100
719  for wflid in wflids:
720  dump = True
721  for key, mrd in self.matrices_.items():
722  for wfl in mrd.workFlows:
723  if wfl.numId == float(wflid):
724  if dump:
725  dump = False
726  print(Fore.GREEN + str(wfl.numId) + Fore.RESET + " " + Fore.YELLOW + wfl.nameId + Fore.RESET)
727  for i,s in enumerate(wfl.cmds):
728  print(fmt % (Fore.RED + str(i+1) + Fore.RESET,
729  (str(s)+' ')))
730  print("\nWorkflow found in %s." % key)
731  else:
732  print("Workflow also found in %s." % key)
733 
734  do_EOF = do_exit
735 
736  TheMatrix(opt).cmdloop()
737  sys.exit(0)
738 
739  if opt.raw and opt.show:
740  ret = showRaw(opt)
741  else:
742  ret = runSelected(opt)
743 
744 
745  sys.exit(ret)
ALCARECOEcalPhiSym_cff.float
float
Definition: ALCARECOEcalPhiSym_cff.py:32
MatrixInjector.performInjectionOptionTest
def performInjectionOptionTest(opt)
Definition: MatrixInjector.py:12
runTheMatrix.runSelected
def runSelected(opt)
Definition: runTheMatrix.py:20
runTheMatrix.TheMatrix.do_predefined
def do_predefined(self, arg)
Definition: runTheMatrix.py:526
runTheMatrix.TheMatrix.do_showWorkflow
def do_showWorkflow(self, arg)
Definition: runTheMatrix.py:550
runTheMatrix.TheMatrix.complete_showWorkflow
def complete_showWorkflow(self, text, line, start_idx, end_idx)
Definition: runTheMatrix.py:544
runTheMatrix.TheMatrix.do_search
def do_search(self, arg)
Definition: runTheMatrix.py:699
runTheMatrix.showRaw
def showRaw(opt)
Definition: runTheMatrix.py:11
runTheMatrix.TheMatrix.do_clear
def do_clear(self, arg)
Definition: runTheMatrix.py:497
runTheMatrix.TheMatrix.help_dumpWorkflowId
def help_dumpWorkflowId(self)
Definition: runTheMatrix.py:707
runTheMatrix.TheMatrix.complete_predefined
def complete_predefined(self, text, line, start_idx, end_idx)
Definition: runTheMatrix.py:520
runTheMatrix.TheMatrix.help_searchInWorkflow
def help_searchInWorkflow(self)
Definition: runTheMatrix.py:659
runTheMatrix.TheMatrix.do_dumpWorkflowId
def do_dumpWorkflowId(self, arg)
Definition: runTheMatrix.py:711
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
genParticles_cff.map
map
Definition: genParticles_cff.py:11
runTheMatrix.TheMatrix.__init__
def __init__(self, opt)
Definition: runTheMatrix.py:483
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:29
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:21
runTheMatrix.TheMatrix.do_exit
def do_exit(self, arg)
Definition: runTheMatrix.py:501
runTheMatrix.TheMatrix.default
def default(self, inp)
Definition: runTheMatrix.py:505
runTheMatrix.TheMatrix.complete_searchInWorkflow
def complete_searchInWorkflow(self, text, line, start_idx, end_idx)
Definition: runTheMatrix.py:664
runTheMatrix.stepOrIndex
def stepOrIndex(s)
Definition: runTheMatrix.py:440
runTheMatrix.TheMatrix.complete_runWorkflow
def complete_runWorkflow(self, text, line, start_idx, end_idx)
Definition: runTheMatrix.py:620
runTheMatrix.TheMatrix.do_runWorkflow
def do_runWorkflow(self, arg)
Definition: runTheMatrix.py:567
str
#define str(s)
Definition: TestProcessor.cc:56
runTheMatrix.TheMatrix.do_jobs
def do_jobs(self, args)
Definition: runTheMatrix.py:646
runTheMatrix.TheMatrix.do_searchInWorkflow
def do_searchInWorkflow(self, arg)
Definition: runTheMatrix.py:670
Generated for CMSSW Reference Manual by doxygen 1.8.14