|
|
|
The main building routines
Definition at line 153 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::__init__ | ( | self, | |
| options, | |||
process = None, |
|||
with_output = False, |
|||
with_input = False |
|||
| ) |
options taken from old cmsDriver and optparse
Definition at line 156 of file ConfigBuilder.py.
00157 : 00158 """options taken from old cmsDriver and optparse """ 00159 00160 options.outfile_name = options.dirout+options.fileout 00161 00162 self._options = options 00163 00164 if self._options.isData and options.isMC: 00165 raise Exception("ERROR: You may specify only --data or --mc, not both") 00166 #if not self._options.conditions: 00167 # raise Exception("ERROR: No conditions given!\nPlease specify conditions. E.g. via --conditions=IDEAL_30X::All") 00168 00169 if hasattr(self._options,"datatier") and self._options.datatier and 'DQMROOT' in self._options.datatier and 'ENDJOB' in self._options.step: 00170 self._options.step=self._options.step.replace(',ENDJOB','') 00171 00172 # what steps are provided by this class? 00173 stepList = [re.sub(r'^prepare_', '', methodName) for methodName in ConfigBuilder.__dict__ if methodName.startswith('prepare_')] 00174 self.stepMap={} 00175 self.stepKeys=[] 00176 for step in self._options.step.split(","): 00177 if step=='': continue 00178 stepParts = step.split(":") 00179 stepName = stepParts[0] 00180 if stepName not in stepList and not stepName.startswith('re'): 00181 raise ValueError("Step "+stepName+" unknown") 00182 if len(stepParts)==1: 00183 self.stepMap[stepName]="" 00184 elif len(stepParts)==2: 00185 self.stepMap[stepName]=stepParts[1].split('+') 00186 elif len(stepParts)==3: 00187 self.stepMap[stepName]=(stepParts[2].split('+'),stepParts[1]) 00188 else: 00189 raise ValueError("Step definition "+step+" invalid") 00190 self.stepKeys.append(stepName) 00191 00192 #print "map of steps is:",self.stepMap 00193 00194 self.with_output = with_output 00195 if hasattr(self._options,"no_output_flag") and self._options.no_output_flag: 00196 self.with_output = False 00197 self.with_input = with_input 00198 if process == None: 00199 self.process = cms.Process(self._options.name) 00200 else: 00201 self.process = process 00202 self.imports = [] 00203 self.define_Configs() 00204 self.schedule = list() 00205 00206 # we are doing three things here: 00207 # creating a process to catch errors 00208 # building the code to re-create the process 00209 00210 00211 00212 self.additionalCommands = [] 00213 # TODO: maybe a list of to be dumped objects would help as well 00214 self.blacklist_paths = [] 00215 self.addedObjects = [] 00216 self.additionalOutputs = {} 00217 00218 self.productionFilterSequence = None 00219 self.nextScheduleIsConditional=False 00220 self.conditionalPaths=[]
| def ConfigBuilder::ConfigBuilder::addCommon | ( | self | ) |
Definition at line 285 of file ConfigBuilder.py.
00286 : 00287 if 'HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys(): 00288 self.process.options = cms.untracked.PSet( Rethrow = cms.untracked.vstring('ProductNotFound'),fileMode = cms.untracked.string('FULLMERGE')) 00289 else: 00290 self.process.options = cms.untracked.PSet( ) 00291 self.addedObjects.append(("","options")) 00292 00293 if self._options.lazy_download: 00294 self.process.AdaptorConfig = cms.Service("AdaptorConfig", 00295 stats = cms.untracked.bool(True), 00296 enable = cms.untracked.bool(True), 00297 cacheHint = cms.untracked.string("lazy-download"), 00298 readHint = cms.untracked.string("read-ahead-buffered") 00299 ) 00300 self.addedObjects.append(("Setup lazy download","AdaptorConfig")) 00301 00302 #self.process.cmsDriverCommand = cms.untracked.PSet( command=cms.untracked.string('cmsDriver.py '+self._options.arguments) ) 00303 #self.addedObjects.append(("what cmsDriver command was used","cmsDriverCommand")) 00304 00305 if self._options.profile: 00306 (start, interval, eventFormat, jobFormat)=self.profileOptions() 00307 self.process.IgProfService = cms.Service("IgProfService", 00308 reportFirstEvent = cms.untracked.int32(start), 00309 reportEventInterval = cms.untracked.int32(interval), 00310 reportToFileAtPostEvent = cms.untracked.string("| gzip -c > %s"%(eventFormat)), 00311 reportToFileAtPostEndJob = cms.untracked.string("| gzip -c > %s"%(jobFormat))) 00312 self.addedObjects.append(("Setup IGProf Service for profiling","IgProfService"))
| def ConfigBuilder::ConfigBuilder::addConditions | ( | self | ) |
Add conditions to the process
Definition at line 714 of file ConfigBuilder.py.
00715 : 00716 """Add conditions to the process""" 00717 if not self._options.conditions: return 00718 00719 if 'FrontierConditions_GlobalTag' in self._options.conditions: 00720 print 'using FrontierConditions_GlobalTag in --conditions is not necessary anymore and will be deprecated soon. please update your command line' 00721 self._options.conditions = self._options.conditions.replace("FrontierConditions_GlobalTag,",'') 00722 00723 self.loadAndRemember(self.ConditionsDefaultCFF) 00724 00725 from Configuration.AlCa.GlobalTag import GlobalTag 00726 self.process.GlobalTag = GlobalTag(self.process.GlobalTag, self._options.conditions, self._options.custom_conditions) 00727 self.additionalCommands.append('from Configuration.AlCa.GlobalTag import GlobalTag') 00728 self.additionalCommands.append('process.GlobalTag = GlobalTag(process.GlobalTag, %s, %s)' % (repr(self._options.conditions), repr(self._options.custom_conditions))) 00729 00730 if self._options.slhc: 00731 self.loadAndRemember("SLHCUpgradeSimulations/Geometry/fakeConditions_%s_cff"%(self._options.slhc,)) 00732
| def ConfigBuilder::ConfigBuilder::addCustomise | ( | self | ) |
Include the customise code
Definition at line 733 of file ConfigBuilder.py.
00734 : 00735 """Include the customise code """ 00736 00737 custOpt=self._options.customisation_file.split(",") 00738 custMap={} 00739 for opt in custOpt: 00740 if opt=='': continue 00741 if opt.count('.')>1: 00742 raise Exception("more than . in the specification:"+opt) 00743 fileName=opt.split('.')[0] 00744 if opt.count('.')==0: rest='customise' 00745 else: 00746 rest=opt.split('.')[1] 00747 if rest=='py': rest='customise' #catch the case of --customise file.py 00748 00749 if fileName in custMap: 00750 custMap[fileName].extend(rest.split('+')) 00751 else: 00752 custMap[fileName]=rest.split('+') 00753 00754 if len(custMap)==0: 00755 final_snippet='\n' 00756 else: 00757 final_snippet='\n# customisation of the process.\n' 00758 00759 allFcn=[] 00760 for opt in custMap: 00761 allFcn.extend(custMap[opt]) 00762 for fcn in allFcn: 00763 if allFcn.count(fcn)!=1: 00764 raise Exception("cannot specify twice "+fcn+" as a customisation method") 00765 00766 for f in custMap: 00767 # let python search for that package and do syntax checking at the same time 00768 packageName = f.replace(".py","").replace("/",".") 00769 __import__(packageName) 00770 package = sys.modules[packageName] 00771 00772 # now ask the package for its definition and pick .py instead of .pyc 00773 customiseFile = re.sub(r'\.pyc$', '.py', package.__file__) 00774 00775 final_snippet+='\n# Automatic addition of the customisation function from '+packageName+'\n' 00776 if self._options.inline_custom: 00777 for line in file(customiseFile,'r'): 00778 if "import FWCore.ParameterSet.Config" in line: 00779 continue 00780 final_snippet += line 00781 else: 00782 final_snippet += 'from %s import %s \n'%(packageName,','.join(custMap[f])) 00783 for fcn in custMap[f]: 00784 print "customising the process with",fcn,"from",f 00785 if not hasattr(package,fcn): 00786 #bound to fail at run time 00787 raise Exception("config "+f+" has no function "+fcn) 00788 #execute the command 00789 self.process=getattr(package,fcn)(self.process) 00790 #and print it in the configuration 00791 final_snippet += "\n#call to customisation function "+fcn+" imported from "+packageName 00792 final_snippet += "\nprocess = %s(process)\n"%(fcn,) 00793 00794 if len(custMap)!=0: 00795 final_snippet += '\n# End of customisation functions\n' 00796 00797 ### now for a usuful command 00798 if self._options.customise_commands: 00799 import string 00800 final_snippet +='\n# Customisation from command line' 00801 for com in self._options.customise_commands.split('\\n'): 00802 com=string.lstrip(com) 00803 self.executeAndRemember(com) 00804 final_snippet +='\n'+com 00805 00806 return final_snippet
| def ConfigBuilder::ConfigBuilder::addExtraStream | ( | self, | |
| name, | |||
| stream, | |||
workflow = 'full' |
|||
| ) |
Definition at line 1035 of file ConfigBuilder.py.
01036 : 01037 # define output module and go from there 01038 output = cms.OutputModule("PoolOutputModule") 01039 if stream.selectEvents.parameters_().__len__()!=0: 01040 output.SelectEvents = stream.selectEvents 01041 else: 01042 output.SelectEvents = cms.untracked.PSet() 01043 output.SelectEvents.SelectEvents=cms.vstring() 01044 if isinstance(stream.paths,tuple): 01045 for path in stream.paths: 01046 output.SelectEvents.SelectEvents.append(path.label()) 01047 else: 01048 output.SelectEvents.SelectEvents.append(stream.paths.label()) 01049 01050 01051 01052 if isinstance(stream.content,str): 01053 evtPset=getattr(self.process,stream.content) 01054 for p in evtPset.parameters_(): 01055 setattr(output,p,getattr(evtPset,p)) 01056 if not self._options.inlineEventContent: 01057 def doNotInlineEventContent(instance,label = "process."+stream.content+".outputCommands"): 01058 return label 01059 output.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 01060 else: 01061 output.outputCommands = stream.content 01062 01063 01064 output.fileName = cms.untracked.string(self._options.dirout+stream.name+'.root') 01065 01066 output.dataset = cms.untracked.PSet( dataTier = stream.dataTier, 01067 filterName = cms.untracked.string(stream.name)) 01068 01069 if self._options.filtername: 01070 output.dataset.filterName= cms.untracked.string(self._options.filtername+"_"+stream.name) 01071 01072 #add an automatic flushing to limit memory consumption 01073 output.eventAutoFlushCompressedSize=cms.untracked.int32(5*1024*1024) 01074 01075 if workflow in ("producers,full"): 01076 if isinstance(stream.paths,tuple): 01077 for path in stream.paths: 01078 self.schedule.append(path) 01079 else: 01080 self.schedule.append(stream.paths) 01081 01082 01083 # in case of relvals we don't want to have additional outputs 01084 if (not self._options.relval) and workflow in ("full","output"): 01085 self.additionalOutputs[name] = output 01086 setattr(self.process,name,output) 01087 01088 if workflow == 'output': 01089 # adjust the select events to the proper trigger results from previous process 01090 filterList = output.SelectEvents.SelectEvents 01091 for i, filter in enumerate(filterList): 01092 filterList[i] = filter+":"+self._options.triggerResultsProcess 01093 01094 return output
| def ConfigBuilder::ConfigBuilder::addMaxEvents | ( | self | ) |
Here we decide how many evts will be processed
Definition at line 313 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::addOutput | ( | self | ) |
Add output module to the process
Definition at line 431 of file ConfigBuilder.py.
00432 : 00433 """ Add output module to the process """ 00434 result="" 00435 if self._options.outputDefinition: 00436 if self._options.datatier: 00437 print "--datatier & --eventcontent options ignored" 00438 00439 def anyOf(listOfKeys,dict,opt=None): 00440 for k in listOfKeys: 00441 if k in dict: 00442 toReturn=dict[k] 00443 dict.pop(k) 00444 return toReturn 00445 if opt!=None: 00446 return opt 00447 else: 00448 raise Exception("any of "+','.join(listOfKeys)+" are mandatory entries of --output options") 00449 00450 #new output convention with a list of dict 00451 outList = eval(self._options.outputDefinition) 00452 for (id,outDefDict) in enumerate(outList): 00453 outDefDictStr=outDefDict.__str__() 00454 if not isinstance(outDefDict,dict): 00455 raise Exception("--output needs to be passed a list of dict"+self._options.outputDefinition+" is invalid") 00456 #requires option: tier 00457 theTier=anyOf(['t','tier','dataTier'],outDefDict) 00458 #optional option: eventcontent, filtername, selectEvents, moduleLabel, filename 00459 ## event content 00460 theStreamType=anyOf(['e','ec','eventContent','streamType'],outDefDict,theTier) 00461 theFilterName=anyOf(['f','ftN','filterName'],outDefDict,'') 00462 theSelectEvent=anyOf(['s','sE','selectEvents'],outDefDict,'') 00463 theModuleLabel=anyOf(['l','mL','moduleLabel'],outDefDict,'') 00464 theExtraOutputCommands=anyOf(['o','oC','outputCommands'],outDefDict,'') 00465 # module label has a particular role 00466 if not theModuleLabel: 00467 tryNames=[theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+'output', 00468 theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+'output', 00469 theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+theSelectEvent.split(',')[0].replace(':','for').replace(' ','')+'output' 00470 ] 00471 for name in tryNames: 00472 if not hasattr(self.process,name): 00473 theModuleLabel=name 00474 break 00475 if not theModuleLabel: 00476 raise Exception("cannot find a module label for specification: "+outDefDictStr) 00477 if id==0: 00478 defaultFileName=self._options.outfile_name 00479 else: 00480 defaultFileName=self._options.outfile_name.replace('.root','_in'+theTier+'.root') 00481 00482 theFileName=self._options.dirout+anyOf(['fn','fileName'],outDefDict,defaultFileName) 00483 if not theFileName.endswith('.root'): 00484 theFileName+='.root' 00485 00486 if len(outDefDict.keys()): 00487 raise Exception("unused keys from --output options: "+','.join(outDefDict.keys())) 00488 if theStreamType=='DQMROOT': theStreamType='DQM' 00489 if theStreamType=='ALL': 00490 theEventContent = cms.PSet(outputCommands = cms.untracked.vstring('keep *')) 00491 else: 00492 theEventContent = getattr(self.process, theStreamType+"EventContent") 00493 00494 if theStreamType=='ALCARECO' and not theFilterName: 00495 theFilterName='StreamALCACombined' 00496 00497 CppType='PoolOutputModule' 00498 if theStreamType=='DQM' and theTier=='DQMROOT': CppType='DQMRootOutputModule' 00499 output = cms.OutputModule(CppType, 00500 theEventContent.clone(), 00501 fileName = cms.untracked.string(theFileName), 00502 dataset = cms.untracked.PSet( 00503 dataTier = cms.untracked.string(theTier), 00504 filterName = cms.untracked.string(theFilterName)) 00505 ) 00506 if not theSelectEvent and hasattr(self.process,'generation_step'): 00507 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step')) 00508 if not theSelectEvent and hasattr(self.process,'filtering_step'): 00509 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step')) 00510 if theSelectEvent: 00511 output.SelectEvents =cms.untracked.PSet(SelectEvents = cms.vstring(theSelectEvent)) 00512 00513 if hasattr(self.process,theModuleLabel): 00514 raise Exception("the current process already has a module "+theModuleLabel+" defined") 00515 #print "creating output module ",theModuleLabel 00516 setattr(self.process,theModuleLabel,output) 00517 outputModule=getattr(self.process,theModuleLabel) 00518 setattr(self.process,theModuleLabel+'_step',cms.EndPath(outputModule)) 00519 path=getattr(self.process,theModuleLabel+'_step') 00520 self.schedule.append(path) 00521 00522 if not self._options.inlineEventContent and hasattr(self.process,theStreamType+"EventContent"): 00523 def doNotInlineEventContent(instance,label = "cms.untracked.vstring(process."+theStreamType+"EventContent.outputCommands)"): 00524 return label 00525 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 00526 if theExtraOutputCommands: 00527 if not isinstance(theExtraOutputCommands,list): 00528 raise Exception("extra ouput command in --option must be a list of strings") 00529 if hasattr(self.process,theStreamType+"EventContent"): 00530 self.executeAndRemember('process.%s.outputCommands.extend(%s)'%(theModuleLabel,theExtraOutputCommands)) 00531 else: 00532 outputModule.outputCommands.extend(theExtraOutputCommands) 00533 00534 result+="\nprocess."+theModuleLabel+" = "+outputModule.dumpPython() 00535 00536 ##ends the --output options model 00537 return result 00538 00539 streamTypes=self._options.eventcontent.split(',') 00540 tiers=self._options.datatier.split(',') 00541 if not self._options.outputDefinition and len(streamTypes)!=len(tiers): 00542 raise Exception("number of event content arguments does not match number of datatier arguments") 00543 00544 # if the only step is alca we don't need to put in an output 00545 if self._options.step.split(',')[0].split(':')[0] == 'ALCA': 00546 return "\n" 00547 00548 for i,(streamType,tier) in enumerate(zip(streamTypes,tiers)): 00549 if streamType=='': continue 00550 if streamType=='DQMROOT': streamType='DQM' 00551 theEventContent = getattr(self.process, streamType+"EventContent") 00552 if i==0: 00553 theFileName=self._options.outfile_name 00554 theFilterName=self._options.filtername 00555 else: 00556 theFileName=self._options.outfile_name.replace('.root','_in'+streamType+'.root') 00557 theFilterName=self._options.filtername 00558 CppType='PoolOutputModule' 00559 if streamType=='DQM' and tier=='DQMROOT': CppType='DQMRootOutputModule' 00560 output = cms.OutputModule(CppType, 00561 theEventContent, 00562 fileName = cms.untracked.string(theFileName), 00563 dataset = cms.untracked.PSet(dataTier = cms.untracked.string(tier), 00564 filterName = cms.untracked.string(theFilterName) 00565 ) 00566 ) 00567 if hasattr(self.process,"generation_step"): 00568 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step')) 00569 if hasattr(self.process,"filtering_step"): 00570 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step')) 00571 00572 if streamType=='ALCARECO': 00573 output.dataset.filterName = cms.untracked.string('StreamALCACombined') 00574 00575 outputModuleName=streamType+'output' 00576 setattr(self.process,outputModuleName,output) 00577 outputModule=getattr(self.process,outputModuleName) 00578 setattr(self.process,outputModuleName+'_step',cms.EndPath(outputModule)) 00579 path=getattr(self.process,outputModuleName+'_step') 00580 self.schedule.append(path) 00581 00582 if not self._options.inlineEventContent: 00583 def doNotInlineEventContent(instance,label = "process."+streamType+"EventContent.outputCommands"): 00584 return label 00585 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 00586 00587 result+="\nprocess."+outputModuleName+" = "+outputModule.dumpPython() 00588 00589 return result
| def ConfigBuilder::ConfigBuilder::addSource | ( | self | ) |
Here the source is built. Priority: file, generator
Definition at line 318 of file ConfigBuilder.py.
00319 : 00320 """Here the source is built. Priority: file, generator""" 00321 self.addedObjects.append(("Input source","source")) 00322 00323 def filesFromOption(self): 00324 for entry in self._options.filein.split(','): 00325 print "entry",entry 00326 if entry.startswith("filelist:"): 00327 filesFromList(entry[9:],self.process.source) 00328 elif entry.startswith("dbs:"): 00329 filesFromDBSQuery('find file where dataset = %s'%(entry[4:]),self.process.source) 00330 else: 00331 self.process.source.fileNames.append(self._options.dirin+entry) 00332 if self._options.secondfilein: 00333 if not hasattr(self.process.source,"secondaryFileNames"): 00334 raise Exception("--secondfilein not compatible with "+self._options.filetype+"input type") 00335 for entry in self._options.secondfilein.split(','): 00336 print "entry",entry 00337 if entry.startswith("filelist:"): 00338 self.process.source.secondaryFileNames.extend((filesFromList(entry[9:]))[0]) 00339 elif entry.startswith("dbs:"): 00340 self.process.source.secondaryFileNames.extend((filesFromDBSQuery('find file where dataset = %s'%(entry[4:])))[0]) 00341 else: 00342 self.process.source.secondaryFileNames.append(self._options.dirin+entry) 00343 00344 if self._options.filein or self._options.dbsquery: 00345 if self._options.filetype == "EDM": 00346 self.process.source=cms.Source("PoolSource", 00347 fileNames = cms.untracked.vstring(), 00348 secondaryFileNames= cms.untracked.vstring()) 00349 filesFromOption(self) 00350 elif self._options.filetype == "DAT": 00351 self.process.source=cms.Source("NewEventStreamFileReader",fileNames = cms.untracked.vstring()) 00352 filesFromOption(self) 00353 elif self._options.filetype == "LHE": 00354 self.process.source=cms.Source("LHESource", fileNames = cms.untracked.vstring()) 00355 if self._options.filein.startswith("lhe:"): 00356 #list the article directory automatically 00357 args=self._options.filein.split(':') 00358 article=args[1] 00359 print 'LHE input from article ',article 00360 location='/store/lhe/' 00361 import os 00362 textOfFiles=os.popen('cmsLHEtoEOSManager.py -l '+article) 00363 for line in textOfFiles: 00364 for fileName in [x for x in line.split() if '.lhe' in x]: 00365 self.process.source.fileNames.append(location+article+'/'+fileName) 00366 if len(args)>2: 00367 self.process.source.skipEvents = cms.untracked.uint32(int(args[2])) 00368 else: 00369 filesFromOption(self) 00370 00371 00372 elif self._options.filetype == "DQM": 00373 self.process.source=cms.Source("DQMRootSource", 00374 fileNames = cms.untracked.vstring()) 00375 filesFromOption(self) 00376 00377 if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"): 00378 self.process.source.processingMode = cms.untracked.string("RunsAndLumis") 00379 00380 if self._options.dbsquery!='': 00381 self.process.source=cms.Source("PoolSource", fileNames = cms.untracked.vstring(),secondaryFileNames = cms.untracked.vstring()) 00382 filesFromDBSQuery(self._options.dbsquery,self.process.source) 00383 00384 if self._options.inputCommands: 00385 if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring() 00386 for command in self._options.inputCommands.split(','): 00387 # remove whitespace around the keep/drop statements 00388 command = command.strip() 00389 if command=='': continue 00390 self.process.source.inputCommands.append(command) 00391 if not self._options.dropDescendant: 00392 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False) 00393 00394 00395 00396 if self._options.lumiToProcess: 00397 import FWCore.PythonUtilities.LumiList as LumiList 00398 self.process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange( LumiList.LumiList(self._options.lumiToProcess).getCMSSWString().split(',') ) 00399 00400 if 'GEN' in self.stepMap or 'LHE' in self.stepMap or (not self._options.filein and hasattr(self._options, "evt_type")): 00401 if self.process.source is None: 00402 self.process.source=cms.Source("EmptySource") 00403 # if option himix is active, drop possibly duplicate DIGI-RAW info: 00404 if self._options.himix==True: 00405 self.process.source.inputCommands = cms.untracked.vstring('drop *','keep *_generator_*_*','keep *_g4SimHits_*_*') 00406 self.process.source.dropDescendantsOfDroppedBranches=cms.untracked.bool(False) 00407 00408 # modify source in case of run-dependent MC 00409 00410 self.runsAndWeights=None 00411 if self._options.runsAndWeightsForMC or self._options.runsScenarioForMC : 00412 if not self._options.isMC : 00413 raise Exception("options --runsAndWeightsForMC and --runsScenarioForMC are only valid for MC") 00414 if self._options.runsAndWeightsForMC: 00415 self.runsAndWeights = eval(self._options.runsAndWeightsForMC) 00416 else: 00417 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights 00418 if type(RunsAndWeights[self._options.runsScenarioForMC])==str: 00419 __import__(RunsAndWeights[self._options.runsScenarioForMC]) 00420 self.runsAndWeights = sys.modules[RunsAndWeights[self._options.runsScenarioForMC]].runProbabilityDistribution 00421 else: 00422 self.runsAndWeights = RunsAndWeights[self._options.runsScenarioForMC] 00423 00424 if self.runsAndWeights: 00425 import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun 00426 ThrowAndSetRandomRun.throwAndSetRandomRun(self.process.source,self.runsAndWeights) 00427 self.additionalCommands.append('import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun') 00428 self.additionalCommands.append('ThrowAndSetRandomRun.throwAndSetRandomRun(process.source,%s)'%(self.runsAndWeights)) 00429 00430 return
| def ConfigBuilder::ConfigBuilder::addStandardSequences | ( | self | ) |
Add selected standard sequences to the process
Definition at line 590 of file ConfigBuilder.py.
00591 : 00592 """ 00593 Add selected standard sequences to the process 00594 """ 00595 # load the pile up file 00596 if self._options.pileup: 00597 pileupSpec=self._options.pileup.split(',')[0] 00598 from Configuration.StandardSequences.Mixing import Mixing,defineMixing 00599 if not pileupSpec in Mixing and '.' not in pileupSpec and 'file:' not in pileupSpec: 00600 raise Exception(pileupSpec+' is not a know mixing scenario:\n available are: '+'\n'.join(Mixing.keys())) 00601 if '.' in pileupSpec: 00602 mixingDict={'file':pileupSpec} 00603 elif pileupSpec.startswith('file:'): 00604 mixingDict={'file':pileupSpec[5:]} 00605 else: 00606 import copy 00607 mixingDict=copy.copy(Mixing[pileupSpec]) 00608 if len(self._options.pileup.split(','))>1: 00609 mixingDict.update(eval(self._options.pileup[self._options.pileup.find(',')+1:])) 00610 if 'file:' in pileupSpec: 00611 #the file is local 00612 self.process.load(mixingDict['file']) 00613 print "inlining mixing module configuration" 00614 self._options.inlineObjets+=',mix' 00615 else: 00616 self.loadAndRemember(mixingDict['file']) 00617 00618 mixingDict.pop('file') 00619 if self._options.pileup_input: 00620 if self._options.pileup_input.startswith('dbs'): 00621 mixingDict['F']=filesFromDBSQuery('find file where dataset = %s'%(self._options.pileup_input[4:],))[0] 00622 else: 00623 mixingDict['F']=self._options.pileup_input.split(',') 00624 specialization=defineMixing(mixingDict,'FASTSIM' in self.stepMap) 00625 for command in specialization: 00626 self.executeAndRemember(command) 00627 if len(mixingDict)!=0: 00628 raise Exception('unused mixing specification: '+mixingDict.keys().__str__()) 00629 00630 00631 # load the geometry file 00632 try: 00633 if len(self.stepMap): 00634 self.loadAndRemember(self.GeometryCFF) 00635 if 'SIM' in self.stepMap or 'reSIM' in self.stepMap: 00636 self.loadAndRemember(self.SimGeometryCFF) 00637 if self.geometryDBLabel: 00638 self.executeAndRemember('process.XMLFromDBSource.label = cms.string("%s")'%(self.geometryDBLabel)) 00639 except ImportError: 00640 print "Geometry option",self._options.geometry,"unknown." 00641 raise 00642 00643 if len(self.stepMap): 00644 self.loadAndRemember(self.magFieldCFF) 00645 00646 00647 for stepName in self.stepKeys: 00648 stepSpec = self.stepMap[stepName] 00649 print "Step:", stepName,"Spec:",stepSpec 00650 if stepName.startswith('re'): 00651 ##add the corresponding input content 00652 if stepName[2:] not in self._options.donotDropOnInput: 00653 self._options.inputEventContent='%s,%s'%(stepName.upper(),self._options.inputEventContent) 00654 stepName=stepName[2:] 00655 if stepSpec=="": 00656 getattr(self,"prepare_"+stepName)(sequence = getattr(self,stepName+"DefaultSeq")) 00657 elif type(stepSpec)==list: 00658 getattr(self,"prepare_"+stepName)(sequence = '+'.join(stepSpec)) 00659 elif type(stepSpec)==tuple: 00660 getattr(self,"prepare_"+stepName)(sequence = ','.join([stepSpec[1],'+'.join(stepSpec[0])])) 00661 else: 00662 raise ValueError("Invalid step definition") 00663 00664 if self._options.restoreRNDSeeds!=False: 00665 #it is either True, or a process name 00666 if self._options.restoreRNDSeeds==True: 00667 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateLabel=cms.untracked.string("randomEngineStateProducer")') 00668 else: 00669 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateTag=cms.untracked.InputTag("randomEngineStateProducer","","%s")'%(self._options.restoreRNDSeeds)) 00670 if self._options.inputEventContent or self._options.inputCommands: 00671 if self._options.inputCommands: 00672 self._options.inputCommands+='keep *_randomEngineStateProducer_*_*,' 00673 else: 00674 self._options.inputCommands='keep *_randomEngineStateProducer_*_*,'
| def ConfigBuilder::ConfigBuilder::build_production_info | ( | self, | |
| evt_type, | |||
| evtnumber | |||
| ) |
Add useful info for the production.
Definition at line 1858 of file ConfigBuilder.py.
01859 : 01860 """ Add useful info for the production. """ 01861 self.process.configurationMetadata=cms.untracked.PSet\ 01862 (version=cms.untracked.string("$Revision: 1.381.2.28 $"), 01863 name=cms.untracked.string("PyReleaseValidation"), 01864 annotation=cms.untracked.string(evt_type+ " nevts:"+str(evtnumber)) 01865 ) 01866 01867 self.addedObjects.append(("Production Info","configurationMetadata")) 01868
| def ConfigBuilder::ConfigBuilder::completeInputCommand | ( | self | ) |
Definition at line 675 of file ConfigBuilder.py.
00676 : 00677 if self._options.inputEventContent: 00678 import copy 00679 def dropSecondDropStar(iec): 00680 #drop occurence of 'drop *' in the list 00681 count=0 00682 for item in iec: 00683 if item=='drop *': 00684 if count!=0: 00685 iec.remove(item) 00686 count+=1 00687 00688 00689 ## allow comma separated input eventcontent 00690 if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring() 00691 for evct in self._options.inputEventContent.split(','): 00692 if evct=='': continue 00693 theEventContent = getattr(self.process, evct+"EventContent") 00694 if hasattr(theEventContent,'outputCommands'): 00695 self.process.source.inputCommands.extend(copy.copy(theEventContent.outputCommands)) 00696 if hasattr(theEventContent,'inputCommands'): 00697 self.process.source.inputCommands.extend(copy.copy(theEventContent.inputCommands)) 00698 00699 dropSecondDropStar(self.process.source.inputCommands) 00700 00701 if not self._options.dropDescendant: 00702 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False) 00703 00704 #re-order the input commands to put drop first 00705 ## drop is before keep: so just sort 00706 ## commented out to make the reGEN work correctly.. 00707 #self.process.source.inputCommands.sort() 00708 #remove that inconsistency: cannot be dropping * and keeping * 00709 if 'keep *' in self.process.source.inputCommands and 'drop *' in self.process.source.inputCommands: 00710 self.process.source.inputCommands.remove( 'keep *' ) 00711 00712 00713 return
| def ConfigBuilder::ConfigBuilder::define_Configs | ( | self | ) |
Definition at line 811 of file ConfigBuilder.py.
00812 : 00813 if len(self.stepMap): 00814 self.loadAndRemember('Configuration/StandardSequences/Services_cff') 00815 if self._options.particleTable not in defaultOptions.particleTableList: 00816 print 'Invalid particle table provided. Options are:' 00817 print defaultOptions.particleTable 00818 sys.exit(-1) 00819 else: 00820 if len(self.stepMap): 00821 self.loadAndRemember('SimGeneral.HepPDTESSource.'+self._options.particleTable+'_cfi') 00822 00823 self.loadAndRemember('FWCore/MessageService/MessageLogger_cfi') 00824 00825 self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreams_cff" 00826 self.GENDefaultCFF="Configuration/StandardSequences/Generator_cff" 00827 self.SIMDefaultCFF="Configuration/StandardSequences/Sim_cff" 00828 self.DIGIDefaultCFF="Configuration/StandardSequences/Digi_cff" 00829 self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRaw_cff" 00830 self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1Emulator_cff' 00831 self.L1MENUDefaultCFF="Configuration/StandardSequences/L1TriggerDefaultMenu_cff" 00832 self.HLTDefaultCFF="Configuration/StandardSequences/HLTtable_cff" 00833 self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_Data_cff" 00834 self.L1RecoDefaultCFF="Configuration/StandardSequences/L1Reco_cff" 00835 self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_Data_cff" 00836 self.SKIMDefaultCFF="Configuration/StandardSequences/Skims_cff" 00837 self.POSTRECODefaultCFF="Configuration/StandardSequences/PostRecoGenerator_cff" 00838 self.VALIDATIONDefaultCFF="Configuration/StandardSequences/Validation_cff" 00839 self.L1HwValDefaultCFF = "Configuration/StandardSequences/L1HwVal_cff" 00840 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOffline_cff" 00841 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/Harvesting_cff" 00842 self.ALCAHARVESTDefaultCFF="Configuration/StandardSequences/AlCaHarvesting_cff" 00843 self.ENDJOBDefaultCFF="Configuration/StandardSequences/EndOfProcess_cff" 00844 self.ConditionsDefaultCFF = "Configuration/StandardSequences/FrontierConditions_GlobalTag_cff" 00845 self.CFWRITERDefaultCFF = "Configuration/StandardSequences/CrossingFrameWriter_cff" 00846 self.REPACKDefaultCFF="Configuration/StandardSequences/DigiToRaw_Repack_cff" 00847 00848 if "DATAMIX" in self.stepMap.keys(): 00849 self.DATAMIXDefaultCFF="Configuration/StandardSequences/DataMixer"+self._options.datamix+"_cff" 00850 self.DIGIDefaultCFF="Configuration/StandardSequences/DigiDM_cff" 00851 self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRawDM_cff" 00852 self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1EmulatorDM_cff' 00853 00854 self.ALCADefaultSeq=None 00855 self.LHEDefaultSeq='externalLHEProducer' 00856 self.GENDefaultSeq='pgen' 00857 self.SIMDefaultSeq=None 00858 self.DIGIDefaultSeq='pdigi' 00859 self.DATAMIXDefaultSeq=None 00860 self.DIGI2RAWDefaultSeq='DigiToRaw' 00861 self.HLTDefaultSeq='GRun' 00862 self.L1DefaultSeq=None 00863 self.L1REPACKDefaultSeq='GT' 00864 self.HARVESTINGDefaultSeq=None 00865 self.ALCAHARVESTDefaultSeq=None 00866 self.CFWRITERDefaultSeq=None 00867 self.RAW2DIGIDefaultSeq='RawToDigi' 00868 self.L1RecoDefaultSeq='L1Reco' 00869 if 'RAW2DIGI' in self.stepMap and 'RECO' in self.stepMap: 00870 self.RECODefaultSeq='reconstruction' 00871 else: 00872 self.RECODefaultSeq='reconstruction_fromRECO' 00873 00874 self.POSTRECODefaultSeq=None 00875 self.L1HwValDefaultSeq='L1HwVal' 00876 self.DQMDefaultSeq='DQMOffline' 00877 self.FASTSIMDefaultSeq='all' 00878 self.VALIDATIONDefaultSeq='' 00879 self.PATLayer0DefaultSeq='all' 00880 self.ENDJOBDefaultSeq='endOfProcess' 00881 self.REPACKDefaultSeq='DigiToRawRepack' 00882 00883 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContent_cff" 00884 00885 # if fastsim switch event content 00886 if "FASTSIM" in self.stepMap.keys(): 00887 self.GENDefaultSeq='pgen_genonly' 00888 self.EVTCONTDefaultCFF = "FastSimulation/Configuration/EventContent_cff" 00889 self.VALIDATIONDefaultCFF = "FastSimulation.Configuration.Validation_cff" 00890 00891 if not self._options.beamspot: 00892 self._options.beamspot=VtxSmearedDefaultKey 00893 00894 # if its MC then change the raw2digi 00895 if self._options.isMC==True: 00896 self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_cff" 00897 self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_cff" 00898 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineMC_cff" 00899 self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreamsMC_cff" 00900 else: 00901 self._options.beamspot = None 00902 00903 #patch for gen, due to backward incompatibility 00904 if 'reGEN' in self.stepMap: 00905 self.GENDefaultSeq='fixGenInfo' 00906 00907 if self._options.scenario=='cosmics': 00908 self.DIGIDefaultCFF="Configuration/StandardSequences/DigiCosmics_cff" 00909 self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionCosmics_cff" 00910 self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsCosmics_cff" 00911 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentCosmics_cff" 00912 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmics_cff" 00913 if self._options.isMC==True: 00914 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmicsMC_cff" 00915 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingCosmics_cff" 00916 self.RECODefaultSeq='reconstructionCosmics' 00917 self.DQMDefaultSeq='DQMOfflineCosmics' 00918 00919 if self._options.himix: 00920 print "From the presence of the himix option, we have determined that this is heavy ions and will use '--scenario HeavyIons'." 00921 self._options.scenario='HeavyIons' 00922 00923 if self._options.scenario=='HeavyIons': 00924 if not self._options.beamspot: 00925 self._options.beamspot=VtxSmearedHIDefaultKey 00926 self.HLTDefaultSeq = 'HIon' 00927 if not self._options.himix: 00928 self.GENDefaultSeq='pgen_hi' 00929 else: 00930 self.GENDefaultSeq='pgen_himix' 00931 self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationHeavyIons_cff" 00932 self.VALIDATIONDefaultSeq='' 00933 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentHeavyIons_cff" 00934 self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionHeavyIons_cff" 00935 self.RECODefaultSeq='reconstructionHeavyIons' 00936 self.ALCADefaultCFF = "Configuration/StandardSequences/AlCaRecoStreamsHeavyIons_cff" 00937 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIons_cff" 00938 self.DQMDefaultSeq='DQMOfflineHeavyIons' 00939 self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsHeavyIons_cff" 00940 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingHeavyIons_cff" 00941 if self._options.isMC==True: 00942 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIonsMC_cff" 00943 00944 00945 self.RAW2RECODefaultSeq=','.join([self.RAW2DIGIDefaultSeq,self.RECODefaultSeq]) 00946 00947 self.USERDefaultSeq='user' 00948 self.USERDefaultCFF=None 00949 00950 # the magnetic field 00951 if self._options.isData: 00952 if self._options.magField==defaultOptions.magField: 00953 print "magnetic field option forced to: AutoFromDBCurrent" 00954 self._options.magField='AutoFromDBCurrent' 00955 self.magFieldCFF = 'Configuration/StandardSequences/MagneticField_'+self._options.magField.replace('.','')+'_cff' 00956 self.magFieldCFF = self.magFieldCFF.replace("__",'_') 00957 00958 # the geometry 00959 self.GeometryCFF='Configuration/StandardSequences/GeometryRecoDB_cff' 00960 self.geometryDBLabel=None 00961 simGeometry='' 00962 if 'FASTSIM' in self.stepMap: 00963 if 'start' in self._options.conditions.lower(): 00964 self.GeometryCFF='FastSimulation/Configuration/Geometries_START_cff' 00965 else: 00966 self.GeometryCFF='FastSimulation/Configuration/Geometries_MC_cff' 00967 else: 00968 def inGeometryKeys(opt): 00969 from Configuration.StandardSequences.GeometryConf import GeometryConf 00970 if opt in GeometryConf: 00971 return GeometryConf[opt] 00972 else: 00973 return opt 00974 00975 geoms=self._options.geometry.split(',') 00976 if len(geoms)==1: geoms=inGeometryKeys(geoms[0]).split(',') 00977 if len(geoms)==2: 00978 #may specify the reco geometry 00979 if '/' in geoms[1] or '_cff' in geoms[1]: 00980 self.GeometryCFF=geoms[1] 00981 else: 00982 self.GeometryCFF='Configuration/StandardSequences/Geometry'+geoms[1]+'_cff' 00983 00984 if (geoms[0].startswith('DB:')): 00985 self.SimGeometryCFF='Configuration/StandardSequences/GeometrySimDB_cff' 00986 self.geometryDBLabel=geoms[0][3:] 00987 print "with DB:" 00988 else: 00989 if '/' in geoms[0] or '_cff' in geoms[0]: 00990 self.SimGeometryCFF=geoms[0] 00991 else: 00992 simGeometry=geoms[0] 00993 if self._options.gflash==True: 00994 self.SimGeometryCFF='Configuration/StandardSequences/Geometry'+geoms[0]+'GFlash_cff' 00995 else: 00996 self.SimGeometryCFF='Configuration/StandardSequences/Geometry'+geoms[0]+'_cff' 00997 00998 # synchronize the geometry configuration and the FullSimulation sequence to be used 00999 if simGeometry not in defaultOptions.geometryExtendedOptions: 01000 self.SIMDefaultCFF="Configuration/StandardSequences/SimIdeal_cff" 01001 01002 if self._options.scenario=='nocoll' or self._options.scenario=='cosmics': 01003 print simGeometry 01004 self.SIMDefaultCFF="Configuration/StandardSequences/SimNOBEAM_cff" 01005 self._options.beamspot='NoSmear' 01006 01007 01008 # Mixing 01009 if self._options.pileup=='default': 01010 from Configuration.StandardSequences.Mixing import MixingDefaultKey,MixingFSDefaultKey 01011 if 'FASTSIM' in self.stepMap: 01012 self._options.pileup=MixingFSDefaultKey 01013 else: 01014 self._options.pileup=MixingDefaultKey 01015 01016 #not driven by a default cff anymore 01017 if self._options.isData: 01018 self._options.pileup=None 01019 if self._options.isMC==True and self._options.himix==False: 01020 if 'FASTSIM' in self.stepMap: 01021 self._options.pileup='FS_'+self._options.pileup 01022 elif self._options.isMC==True and self._options.himix==True: 01023 self._options.pileup='HiMix' 01024 01025 01026 if self._options.slhc: 01027 if 'stdgeom' not in self._options.slhc: 01028 self.SimGeometryCFF='SLHCUpgradeSimulations.Geometry.%s_cmsSimIdealGeometryXML_cff'%(self._options.slhc,) 01029 self.DIGIDefaultCFF='SLHCUpgradeSimulations/Geometry/Digi_%s_cff'%(self._options.slhc,) 01030 if self._options.pileup!=defaultOptions.pileup: 01031 self._options.pileup='SLHC_%s_%s'%(self._options.pileup,self._options.slhc) 01032 01033 self.REDIGIDefaultSeq=self.DIGIDefaultSeq
| def ConfigBuilder::ConfigBuilder::executeAndRemember | ( | self, | |
| command | |||
| ) |
helper routine to remember replace statements
Definition at line 278 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::expandMapping | ( | self, | |
| seqList, | |||
| mapping, | |||
index = None |
|||
| ) |
Definition at line 1695 of file ConfigBuilder.py.
01696 : 01697 maxLevel=20 01698 level=0 01699 while '@' in repr(seqList) and level<maxLevel: 01700 level+=1 01701 for specifiedCommand in seqList: 01702 if specifiedCommand.startswith('@'): 01703 location=specifiedCommand[1:] 01704 if not location in mapping: 01705 raise Exception("Impossible to map "+location+" from "+repr(mapping)) 01706 mappedTo=mapping[location] 01707 if index!=None: 01708 mappedTo=mappedTo[index] 01709 seqList.remove(specifiedCommand) 01710 seqList.extend(mappedTo.split('+')) 01711 break; 01712 if level==maxLevel: 01713 raise Exception("Could not fully expand "+repr(seqList)+" from "+repr(mapping))
| def ConfigBuilder::ConfigBuilder::finalizeFastSimHLT | ( | self | ) |
Definition at line 1809 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::load | ( | self, | |
| includeFile | |||
| ) |
Definition at line 264 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::loadAndRemember | ( | self, | |
| includeFile | |||
| ) |
helper routine to load am memorize imports
Definition at line 269 of file ConfigBuilder.py.
00270 : 00271 """helper routine to load am memorize imports""" 00272 # we could make the imports a on-the-fly data method of the process instance itself 00273 # not sure if the latter is a good idea 00274 includeFile = includeFile.replace('/','.') 00275 self.imports.append(includeFile) 00276 self.process.load(includeFile) 00277 return sys.modules[includeFile]
| def ConfigBuilder::ConfigBuilder::loadDefaultOrSpecifiedCFF | ( | self, | |
| sequence, | |||
| defaultCFF | |||
| ) |
Definition at line 1100 of file ConfigBuilder.py.
01101 : 01102 if ( len(sequence.split('.'))==1 ): 01103 l=self.loadAndRemember(defaultCFF) 01104 elif ( len(sequence.split('.'))==2 ): 01105 l=self.loadAndRemember(sequence.split('.')[0]) 01106 sequence=sequence.split('.')[1] 01107 else: 01108 print "sub sequence configuration must be of the form dir/subdir/cff.a+b+c or cff.a" 01109 print sequence,"not recognized" 01110 raise 01111 return l
| def ConfigBuilder::ConfigBuilder::prepare | ( | self, | |
doChecking = False |
|||
| ) |
Prepare the configuration string and add missing pieces.
Definition at line 1869 of file ConfigBuilder.py.
01870 : 01871 """ Prepare the configuration string and add missing pieces.""" 01872 01873 self.loadAndRemember(self.EVTCONTDefaultCFF) #load the event contents regardless 01874 self.addMaxEvents() 01875 if self.with_input: 01876 self.addSource() 01877 self.addStandardSequences() 01878 ##adding standard sequences might change the inputEventContent option and therefore needs to be finalized after 01879 self.completeInputCommand() 01880 01881 self.addConditions() 01882 01883 01884 outputModuleCfgCode="" 01885 if not 'HARVESTING' in self.stepMap.keys() and not 'SKIM' in self.stepMap.keys() and not 'ALCAHARVEST' in self.stepMap.keys() and not 'ALCAOUTPUT' in self.stepMap.keys() and self.with_output: 01886 outputModuleCfgCode=self.addOutput() 01887 01888 self.addCommon() 01889 01890 self.pythonCfgCode = "# Auto generated configuration file\n" 01891 self.pythonCfgCode += "# using: \n# "+__version__[1:-1]+"\n# "+__source__[1:-1]+'\n' 01892 self.pythonCfgCode += "# with command line options: "+self._options.arguments+'\n' 01893 self.pythonCfgCode += "import FWCore.ParameterSet.Config as cms\n\n" 01894 self.pythonCfgCode += "process = cms.Process('"+self.process.name_()+"')\n\n" 01895 01896 self.pythonCfgCode += "# import of standard configurations\n" 01897 for module in self.imports: 01898 self.pythonCfgCode += ("process.load('"+module+"')\n") 01899 01900 # production info 01901 if not hasattr(self.process,"configurationMetadata"): 01902 self.build_production_info(self._options.evt_type, self._options.number) 01903 else: 01904 #the PSet was added via a load 01905 self.addedObjects.append(("Production Info","configurationMetadata")) 01906 01907 self.pythonCfgCode +="\n" 01908 for comment,object in self.addedObjects: 01909 if comment!="": 01910 self.pythonCfgCode += "\n# "+comment+"\n" 01911 self.pythonCfgCode += dumpPython(self.process,object) 01912 01913 # dump the output definition 01914 self.pythonCfgCode += "\n# Output definition\n" 01915 self.pythonCfgCode += outputModuleCfgCode 01916 01917 # dump all additional outputs (e.g. alca or skim streams) 01918 self.pythonCfgCode += "\n# Additional output definition\n" 01919 #I do not understand why the keys are not normally ordered. 01920 nl=self.additionalOutputs.keys() 01921 nl.sort() 01922 for name in nl: 01923 output = self.additionalOutputs[name] 01924 self.pythonCfgCode += "process.%s = %s" %(name, output.dumpPython()) 01925 tmpOut = cms.EndPath(output) 01926 setattr(self.process,name+'OutPath',tmpOut) 01927 self.schedule.append(tmpOut) 01928 01929 # dump all additional commands 01930 self.pythonCfgCode += "\n# Other statements\n" 01931 for command in self.additionalCommands: 01932 self.pythonCfgCode += command + "\n" 01933 01934 #comma separated list of objects that deserve to be inlined in the configuration (typically from a modified config deep down) 01935 for object in self._options.inlineObjets.split(','): 01936 if not object: 01937 continue 01938 if not hasattr(self.process,object): 01939 print 'cannot inline -'+object+'- : not known' 01940 else: 01941 self.pythonCfgCode +='\n' 01942 self.pythonCfgCode +=dumpPython(self.process,object) 01943 01944 # dump all paths 01945 self.pythonCfgCode += "\n# Path and EndPath definitions\n" 01946 for path in self.process.paths: 01947 if getattr(self.process,path) not in self.blacklist_paths: 01948 self.pythonCfgCode += dumpPython(self.process,path) 01949 01950 for endpath in self.process.endpaths: 01951 if getattr(self.process,endpath) not in self.blacklist_paths: 01952 self.pythonCfgCode += dumpPython(self.process,endpath) 01953 01954 # dump the schedule 01955 self.pythonCfgCode += "\n# Schedule definition\n" 01956 result = "process.schedule = cms.Schedule(" 01957 01958 # handling of the schedule 01959 self.process.schedule = cms.Schedule() 01960 for item in self.schedule: 01961 if not isinstance(item, cms.Schedule): 01962 self.process.schedule.append(item) 01963 else: 01964 self.process.schedule.extend(item) 01965 01966 if hasattr(self.process,"HLTSchedule"): 01967 beforeHLT = self.schedule[:self.schedule.index(self.process.HLTSchedule)] 01968 afterHLT = self.schedule[self.schedule.index(self.process.HLTSchedule)+1:] 01969 pathNames = ['process.'+p.label_() for p in beforeHLT] 01970 result += ','.join(pathNames)+')\n' 01971 result += 'process.schedule.extend(process.HLTSchedule)\n' 01972 pathNames = ['process.'+p.label_() for p in afterHLT] 01973 result += 'process.schedule.extend(['+','.join(pathNames)+'])\n' 01974 else: 01975 pathNames = ['process.'+p.label_() for p in self.schedule] 01976 result ='process.schedule = cms.Schedule('+','.join(pathNames)+')\n' 01977 01978 self.pythonCfgCode += result 01979 01980 #repacked version 01981 if self._options.isRepacked: 01982 self.pythonCfgCode +="\n" 01983 self.pythonCfgCode +="from Configuration.PyReleaseValidation.ConfigBuilder import MassReplaceInputTag\n" 01984 self.pythonCfgCode +="MassReplaceInputTag(process)\n" 01985 MassReplaceInputTag(self.process) 01986 01987 # special treatment in case of production filter sequence 2/2 01988 if self.productionFilterSequence: 01989 self.pythonCfgCode +='# filter all path with the production filter sequence\n' 01990 self.pythonCfgCode +='for path in process.paths:\n' 01991 if len(self.conditionalPaths): 01992 self.pythonCfgCode +='\tif not path in %s: continue\n'%str(self.conditionalPaths) 01993 self.pythonCfgCode +='\tgetattr(process,path)._seq = process.%s * getattr(process,path)._seq \n'%(self.productionFilterSequence,) 01994 pfs = getattr(self.process,self.productionFilterSequence) 01995 for path in self.process.paths: 01996 if not path in self.conditionalPaths: continue 01997 getattr(self.process,path)._seq = pfs * getattr(self.process,path)._seq 01998 01999 # dump customise fragment 02000 self.pythonCfgCode += self.addCustomise() 02001 02002 # make the .io file 02003 02004 if self._options.io: 02005 #io=open(self._options.python_filename.replace('.py','.io'),'w') 02006 if not self._options.io.endswith('.io'): self._option.io+='.io' 02007 io=open(self._options.io,'w') 02008 ioJson={} 02009 if hasattr(self.process.source,"fileNames"): 02010 if len(self.process.source.fileNames.value()): 02011 ioJson['primary']=self.process.source.fileNames.value() 02012 if hasattr(self.process.source,"secondaryFileNames"): 02013 if len(self.process.source.secondaryFileNames.value()): 02014 ioJson['secondary']=self.process.source.secondaryFileNames.value() 02015 02016 for (o,om) in self.process.outputModules_().items(): 02017 ioJson[o]=om.fileName.value() 02018 #ioJson['GT']=self._options.conditions 02019 ioJson['GT']=self.process.GlobalTag.globaltag.value() 02020 if self.productionFilterSequence: 02021 ioJson['filter']=self.productionFilterSequence 02022 import json 02023 io.write(json.dumps(ioJson)) 02024 return 02025 02026 02027
| def ConfigBuilder::ConfigBuilder::prepare_ALCA | ( | self, | |
sequence = None, |
|||
workflow = 'full' |
|||
| ) |
Enrich the process with alca streams
Definition at line 1149 of file ConfigBuilder.py.
01150 : 01151 """ Enrich the process with alca streams """ 01152 alcaConfig=self.loadDefaultOrSpecifiedCFF(sequence,self.ALCADefaultCFF) 01153 sequence = sequence.split('.')[-1] 01154 01155 # decide which ALCA paths to use 01156 alcaList = sequence.split("+") 01157 maxLevel=0 01158 from Configuration.AlCa.autoAlca import autoAlca 01159 # support @X from autoAlca.py, and recursion support: i.e T0:@Mu+@EG+... 01160 self.expandMapping(alcaList,autoAlca) 01161 01162 for name in alcaConfig.__dict__: 01163 alcastream = getattr(alcaConfig,name) 01164 shortName = name.replace('ALCARECOStream','') 01165 if shortName in alcaList and isinstance(alcastream,cms.FilteredStream): 01166 output = self.addExtraStream(name,alcastream, workflow = workflow) 01167 if 'DQM' in alcaList: 01168 if not self._options.inlineEventContent and hasattr(self.process,name): 01169 self.executeAndRemember('process.' + name + '.outputCommands.append("keep *_MEtoEDMConverter_*_*")') 01170 else: 01171 output.outputCommands.append("keep *_MEtoEDMConverter_*_*") 01172 01173 #rename the HLT process name in the alca modules 01174 if self._options.hltProcess or 'HLT' in self.stepMap: 01175 if isinstance(alcastream.paths,tuple): 01176 for path in alcastream.paths: 01177 self.renameHLTprocessInSequence(path.label()) 01178 else: 01179 self.renameHLTprocessInSequence(alcastream.paths.label()) 01180 01181 for i in range(alcaList.count(shortName)): 01182 alcaList.remove(shortName) 01183 01184 # DQM needs a special handling 01185 elif name == 'pathALCARECODQM' and 'DQM' in alcaList: 01186 path = getattr(alcaConfig,name) 01187 self.schedule.append(path) 01188 alcaList.remove('DQM') 01189 01190 if isinstance(alcastream,cms.Path): 01191 #black list the alca path so that they do not appear in the cfg 01192 self.blacklist_paths.append(alcastream) 01193 01194 01195 if len(alcaList) != 0: 01196 available=[] 01197 for name in alcaConfig.__dict__: 01198 alcastream = getattr(alcaConfig,name) 01199 if isinstance(alcastream,cms.FilteredStream): 01200 available.append(name.replace('ALCARECOStream','')) 01201 print "The following alcas could not be found "+str(alcaList) 01202 print "available ",available 01203 #print "verify your configuration, ignoring for now" 01204 raise Exception("The following alcas could not be found "+str(alcaList))
| def ConfigBuilder::ConfigBuilder::prepare_ALCAHARVEST | ( | self, | |
sequence = None |
|||
| ) |
Enrich the process with AlCaHarvesting step
Definition at line 1780 of file ConfigBuilder.py.
01781 : 01782 """ Enrich the process with AlCaHarvesting step """ 01783 harvestingConfig = self.loadAndRemember(self.ALCAHARVESTDefaultCFF) 01784 sequence=sequence.split(".")[-1] 01785 01786 # decide which AlcaHARVESTING paths to use 01787 harvestingList = sequence.split("+") 01788 for name in harvestingConfig.__dict__: 01789 harvestingstream = getattr(harvestingConfig,name) 01790 if name in harvestingList and isinstance(harvestingstream,cms.Path): 01791 self.schedule.append(harvestingstream) 01792 self.executeAndRemember("process.PoolDBOutputService.toPut.append(process.ALCAHARVEST" + name + "_dbOutput)") 01793 self.executeAndRemember("process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVEST" + name + "_metadata)") 01794 harvestingList.remove(name) 01795 # append the common part at the end of the sequence 01796 lastStep = getattr(harvestingConfig,"ALCAHARVESTDQMSaveAndMetadataWriter") 01797 self.schedule.append(lastStep) 01798 01799 if len(harvestingList) != 0 and 'dummyHarvesting' not in harvestingList : 01800 print "The following harvesting could not be found : ", harvestingList 01801 raise Exception("The following harvesting could not be found : "+str(harvestingList)) 01802 01803
| def ConfigBuilder::ConfigBuilder::prepare_ALCAOUTPUT | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1146 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_ALCAPRODUCER | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1143 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_CFWRITER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the crossing frame writer step
Definition at line 1328 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DATAMIX | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the digitisation step
Definition at line 1334 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DIGI | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the digitisation step
Definition at line 1312 of file ConfigBuilder.py.
01313 : 01314 """ Enrich the schedule with the digitisation step""" 01315 self.loadDefaultOrSpecifiedCFF(sequence,self.DIGIDefaultCFF) 01316 01317 if self._options.gflash==True: 01318 self.loadAndRemember("Configuration/StandardSequences/GFlashDIGI_cff") 01319 01320 if self._options.himix==True: 01321 self.loadAndRemember("SimGeneral/MixingModule/himixDIGI_cff") 01322 01323 if self._options.restoreRNDSeeds: 01324 self.executeAndRemember("process.mix.playback = True") 01325 01326 self.scheduleSequence(sequence.split('.')[-1],'digitisation_step') 01327 return
| def ConfigBuilder::ConfigBuilder::prepare_DIGI2RAW | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1340 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DQM | ( | self, | |
sequence = 'DQMOffline' |
|||
| ) |
Definition at line 1714 of file ConfigBuilder.py.
01715 : 01716 # this one needs replacement 01717 01718 self.loadDefaultOrSpecifiedCFF(sequence,self.DQMOFFLINEDefaultCFF) 01719 sequenceList=sequence.split('.')[-1].split('+') 01720 from DQMOffline.Configuration.autoDQM import autoDQM 01721 self.expandMapping(sequenceList,autoDQM,index=0) 01722 01723 if len(set(sequenceList))!=len(sequenceList): 01724 sequenceList=list(set(sequenceList)) 01725 print "Duplicate entries for DQM:, using",sequenceList 01726 pathName='dqmoffline_step' 01727 01728 for (i,sequence) in enumerate(sequenceList): 01729 if (i!=0): 01730 pathName='dqmoffline_%d_step'%(i) 01731 01732 if 'HLT' in self.stepMap.keys() or self._options.hltProcess: 01733 self.renameHLTprocessInSequence(sequence) 01734 01735 # if both HLT and DQM are run in the same process, schedule [HLT]DQM in an EndPath 01736 if 'HLT' in self.stepMap.keys(): 01737 # need to put [HLT]DQM in an EndPath, to access the HLT trigger results 01738 #setattr(self.process,pathName, cms.EndPath( getattr(self.process, sequence ) ) ) 01739 self.scheduleSequence(sequence, pathName, what='EndPath') 01740 01741 else: 01742 # schedule DQM as a standard Path 01743 #setattr(self.process,pathName, cms.Path( getattr(self.process, sequence) ) ) 01744 self.scheduleSequence(sequence, pathName, what='Path') 01745 #self.schedule.append(getattr(self.process,pathName)) 01746
| def ConfigBuilder::ConfigBuilder::prepare_ENDJOB | ( | self, | |
sequence = 'endOfProcess' |
|||
| ) |
Definition at line 1804 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_FASTSIM | ( | self, | |
sequence = "all" |
|||
| ) |
Enrich the schedule with fastsim
Definition at line 1813 of file ConfigBuilder.py.
01814 : 01815 """Enrich the schedule with fastsim""" 01816 self.loadAndRemember("FastSimulation/Configuration/FamosSequences_cff") 01817 01818 if sequence in ('all','allWithHLTFiltering',''): 01819 if not 'HLT' in self.stepMap.keys(): 01820 self.prepare_HLT(sequence=None) 01821 01822 self.executeAndRemember("process.famosSimHits.SimulateCalorimetry = True") 01823 self.executeAndRemember("process.famosSimHits.SimulateTracking = True") 01824 01825 self.executeAndRemember("process.simulation = cms.Sequence(process.simulationWithFamos)") 01826 self.executeAndRemember("process.HLTEndSequence = cms.Sequence(process.reconstructionWithFamos)") 01827 01828 # since we have HLT here, the process should be called HLT 01829 self._options.name = "HLT" 01830 01831 # if we don't want to filter after HLT but simulate everything regardless of what HLT tells, we have to add reconstruction explicitly 01832 if sequence == 'all' and not 'HLT' in self.stepMap.keys(): #(a) 01833 self.finalizeFastSimHLT() 01834 elif sequence == 'famosWithEverything': 01835 self.process.fastsim_step = cms.Path( getattr(self.process, "famosWithEverything") ) 01836 self.schedule.append(self.process.fastsim_step) 01837 01838 # now the additional commands we need to make the config work 01839 self.executeAndRemember("process.VolumeBasedMagneticFieldESProducer.useParametrizedTrackerField = True") 01840 else: 01841 print "FastSim setting", sequence, "unknown." 01842 raise ValueError 01843 01844 if 'Flat' in self._options.beamspot: 01845 beamspotType = 'Flat' 01846 elif 'Gauss' in self._options.beamspot: 01847 beamspotType = 'Gaussian' 01848 else: 01849 beamspotType = 'BetaFunc' 01850 self.loadAndRemember('IOMC.EventVertexGenerators.VtxSmearedParameters_cfi') 01851 beamspotName = 'process.%sVtxSmearingParameters' %(self._options.beamspot) 01852 self.executeAndRemember(beamspotName+'.type = cms.string("%s")'%(beamspotType)) 01853 self.executeAndRemember('process.famosSimHits.VertexGenerator = '+beamspotName) 01854 if hasattr(self.process,'famosPileUp'): 01855 self.executeAndRemember('process.famosPileUp.VertexGenerator = '+beamspotName) 01856 01857
| def ConfigBuilder::ConfigBuilder::prepare_FILTER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with a user defined filter sequence
Definition at line 1473 of file ConfigBuilder.py.
01474 : 01475 ''' Enrich the schedule with a user defined filter sequence ''' 01476 ## load the relevant part 01477 filterConfig=self.load(sequence.split('.')[0]) 01478 filterSeq=sequence.split('.')[-1] 01479 ## print it in the configuration 01480 class PrintAllModules(object): 01481 def __init__(self): 01482 self.inliner='' 01483 pass 01484 def enter(self,visitee): 01485 try: 01486 label=visitee.label() 01487 ##needs to be in reverse order 01488 self.inliner=label+','+self.inliner 01489 except: 01490 pass 01491 def leave(self,v): pass 01492 01493 expander=PrintAllModules() 01494 getattr(self.process,filterSeq).visit( expander ) 01495 self._options.inlineObjets+=','+expander.inliner 01496 self._options.inlineObjets+=','+filterSeq 01497 01498 ## put the filtering path in the schedule 01499 self.scheduleSequence(filterSeq,'filtering_step') 01500 self.nextScheduleIsConditional=True 01501 ## put it before all the other paths 01502 self.productionFilterSequence = filterSeq 01503 01504 return
| def ConfigBuilder::ConfigBuilder::prepare_GEN | ( | self, | |
sequence = None |
|||
| ) |
load the fragment of generator configuration
Definition at line 1221 of file ConfigBuilder.py.
01222 : 01223 """ load the fragment of generator configuration """ 01224 loadFailure=False 01225 #remove trailing .py 01226 #support old style .cfi by changing into something.cfi into something_cfi 01227 #remove python/ from the name 01228 loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','') 01229 #standard location of fragments 01230 if not '/' in loadFragment: 01231 loadFragment='Configuration.Generator.'+loadFragment 01232 else: 01233 loadFragment=loadFragment.replace('/','.') 01234 try: 01235 print "Loading generator fragment from",loadFragment 01236 __import__(loadFragment) 01237 except: 01238 loadFailure=True 01239 #if self.process.source and self.process.source.type_()=='EmptySource': 01240 if not (self._options.filein or self._options.dbsquery): 01241 raise Exception("Neither gen fragment of input files provided: this is an inconsistent GEN step configuration") 01242 01243 if not loadFailure: 01244 generatorModule=sys.modules[loadFragment] 01245 genModules=generatorModule.__dict__ 01246 if self._options.hideGen: 01247 self.loadAndRemember(loadFragment) 01248 else: 01249 self.process.load(loadFragment) 01250 # expose the objects from that fragment to the configuration 01251 import FWCore.ParameterSet.Modules as cmstypes 01252 for name in genModules: 01253 theObject = getattr(generatorModule,name) 01254 if isinstance(theObject, cmstypes._Module): 01255 self._options.inlineObjets=name+','+self._options.inlineObjets 01256 elif isinstance(theObject, cms.Sequence) or isinstance(theObject, cmstypes.ESProducer): 01257 self._options.inlineObjets+=','+name 01258 01259 if sequence == self.GENDefaultSeq or sequence == 'pgen_genonly' or ( sequence == 'pgen_hi' and 'generator' in genModules): 01260 if 'ProductionFilterSequence' in genModules and ('generator' in genModules or 'hiSignal' in genModules): 01261 self.productionFilterSequence = 'ProductionFilterSequence' 01262 elif 'generator' in genModules: 01263 self.productionFilterSequence = 'generator' 01264 01265 """ Enrich the schedule with the rest of the generation step """ 01266 self.loadDefaultOrSpecifiedCFF(sequence,self.GENDefaultCFF) 01267 genSeqName=sequence.split('.')[-1] 01268 01269 if not 'FASTSIM' in self.stepMap: 01270 try: 01271 from Configuration.StandardSequences.VtxSmeared import VtxSmeared 01272 self.loadAndRemember(VtxSmeared[self._options.beamspot]) 01273 except ImportError: 01274 raise Exception("VertexSmearing type or beamspot "+self._options.beamspot+" unknown.") 01275 01276 if self._options.scenario == 'HeavyIons' and self._options.himix: 01277 self.loadAndRemember("SimGeneral/MixingModule/himixGEN_cff") 01278 01279 self.process.generation_step = cms.Path( getattr(self.process,genSeqName) ) 01280 self.schedule.append(self.process.generation_step) 01281 01282 #register to the genstepfilter the name of the path (static right now, but might evolve) 01283 self.executeAndRemember('process.genstepfilter.triggerConditions=cms.vstring("generation_step")') 01284 01285 if 'reGEN' in self.stepMap: 01286 #stop here 01287 return 01288 01289 """ Enrich the schedule with the summary of the filter step """ 01290 #the gen filter in the endpath 01291 self.loadAndRemember("GeneratorInterface/Core/genFilterSummary_cff") 01292 self.scheduleSequenceAtEnd('genFilterSummary','genfiltersummary_step') 01293 return
| def ConfigBuilder::ConfigBuilder::prepare_HARVESTING | ( | self, | |
sequence = None |
|||
| ) |
Enrich the process with harvesting step
Definition at line 1747 of file ConfigBuilder.py.
01748 : 01749 """ Enrich the process with harvesting step """ 01750 self.EDMtoMECFF='Configuration/StandardSequences/EDMtoME'+self._options.harvesting+'_cff' 01751 self.loadAndRemember(self.EDMtoMECFF) 01752 self.scheduleSequence('EDMtoME','edmtome_step') 01753 01754 harvestingConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.HARVESTINGDefaultCFF) 01755 sequence = sequence.split('.')[-1] 01756 01757 # decide which HARVESTING paths to use 01758 harvestingList = sequence.split("+") 01759 from DQMOffline.Configuration.autoDQM import autoDQM 01760 self.expandMapping(harvestingList,autoDQM,index=1) 01761 01762 if len(set(harvestingList))!=len(harvestingList): 01763 harvestingList=list(set(harvestingList)) 01764 print "Duplicate entries for HARVESTING, using",harvestingList 01765 01766 for name in harvestingList: 01767 if not name in harvestingConfig.__dict__: 01768 print name,"is not a possible harvesting type. Available are",harvestingConfig.__dict__.keys() 01769 continue 01770 harvestingstream = getattr(harvestingConfig,name) 01771 if isinstance(harvestingstream,cms.Path): 01772 self.schedule.append(harvestingstream) 01773 self.blacklist_paths.append(harvestingstream) 01774 if isinstance(harvestingstream,cms.Sequence): 01775 setattr(self.process,name+"_step",cms.Path(harvestingstream)) 01776 self.schedule.append(getattr(self.process,name+"_step")) 01777 01778 self.scheduleSequence('DQMSaver','dqmsave_step') 01779 return
| def ConfigBuilder::ConfigBuilder::prepare_HLT | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the HLT simulation step
Definition at line 1375 of file ConfigBuilder.py.
01376 : 01377 """ Enrich the schedule with the HLT simulation step""" 01378 if not sequence: 01379 print "no specification of the hlt menu has been given, should never happen" 01380 raise Exception('no HLT sequence provided') 01381 01382 if '@' in sequence: 01383 # case where HLT:@something was provided 01384 from Configuration.HLT.autoHLT import autoHLT 01385 key = sequence[1:] 01386 if key in autoHLT: 01387 sequence = autoHLT[key] 01388 else: 01389 raise ValueError('no HLT mapping key "%s" found in autoHLT' % key) 01390 01391 if ',' in sequence: 01392 # case where HLT:something:something was provided transcribed in sequence = something,something 01393 self.executeAndRemember('import HLTrigger.Configuration.Utilities') 01394 optionsForHLT = {} 01395 if self._options.scenario == 'HeavyIons': 01396 optionsForHLT['type'] = 'HIon' 01397 else: 01398 optionsForHLT['type'] = 'GRun' 01399 optionsForHLTConfig = ', '.join('%s=%s' % (key, repr(val)) for (key, val) in optionsForHLT.iteritems()) 01400 if sequence == 'run,fromSource': 01401 if hasattr(self.process.source,'firstRun'): 01402 self.executeAndRemember('process.loadHltConfiguration("run:%%d" %% (process.source.firstRun.value()), %s)' % (optionsForHLTConfig)) 01403 elif hasattr(self.process.source,'setRunNumber'): 01404 self.executeAndRemember('process.loadHltConfiguration("run:%%d" %% (process.source.setRunNumber.value()), %s)' % (optionsForHLTConfig)) 01405 else: 01406 raise Exception('Cannot replace menu to load %s'%(sequence)) 01407 elif sequence == 'cached,fromSource': 01408 self.executeAndRemember('import Configuration.HLT.cachedHLT') 01409 if hasattr(self.process.source,'firstRun'): 01410 self.executeAndRemember('process.loadCachedHltConfiguration( process.source.firstRun.value(), %s )' % ('FASTSIM' in self.stepMap)) 01411 elif hasattr(self.process.source,'setRunNumber'): 01412 self.executeAndRemember('process.loadCachedHltConfiguration( process.source.setRunNumber.value(), %s )' % ('FASTSIM' in self.stepMap)) 01413 else: 01414 raise Exception('Cannot replace menu to load %s'%(sequence)) 01415 elif sequence.startswith( 'cached,' ): 01416 run = int( sequence.split(',')[1] ) 01417 self.executeAndRemember('import Configuration.HLT.cachedHLT') 01418 self.executeAndRemember('process.loadCachedHltConfiguration( %d, %s )' % (run, 'FASTSIM' in self.stepMap)) 01419 else: 01420 self.executeAndRemember('process.loadHltConfiguration("%s",%s)'%(sequence.replace(',',':'),optionsForHLTConfig)) 01421 else: 01422 if 'FASTSIM' in self.stepMap: 01423 self.loadAndRemember('HLTrigger/Configuration/HLT_%s_Famos_cff' % sequence) 01424 else: 01425 self.loadAndRemember('HLTrigger/Configuration/HLT_%s_cff' % sequence) 01426 01427 if self._options.isMC: 01428 self._options.customisation_file+=",HLTrigger/Configuration/customizeHLTforMC.customizeHLTforMC" 01429 01430 if self._options.name != 'HLT': 01431 self.additionalCommands.append('from HLTrigger.Configuration.CustomConfigs import ProcessName') 01432 self.additionalCommands.append('process = ProcessName(process)') 01433 self.additionalCommands.append('') 01434 from HLTrigger.Configuration.CustomConfigs import ProcessName 01435 self.process = ProcessName(self.process) 01436 01437 self.schedule.append(self.process.HLTSchedule) 01438 [self.blacklist_paths.append(path) for path in self.process.HLTSchedule if isinstance(path,(cms.Path,cms.EndPath))] 01439 if ('FASTSIM' in self.stepMap and 'HLT' in self.stepMap): 01440 self.finalizeFastSimHLT() 01441
| def ConfigBuilder::ConfigBuilder::prepare_L1 | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the L1 simulation step
Definition at line 1350 of file ConfigBuilder.py.
01351 : 01352 """ Enrich the schedule with the L1 simulation step""" 01353 if not sequence: 01354 self.loadAndRemember(self.L1EMDefaultCFF) 01355 else: 01356 # let the L1 package decide for the scenarios available 01357 from L1Trigger.Configuration.ConfigBuilder import getConfigsForScenario 01358 listOfImports = getConfigsForScenario(sequence) 01359 for file in listOfImports: 01360 self.loadAndRemember(file) 01361 self.scheduleSequence('SimL1Emulator','L1simulation_step') 01362 return
| def ConfigBuilder::ConfigBuilder::prepare_L1HwVal | ( | self, | |
sequence = 'L1HwVal' |
|||
| ) |
Enrich the schedule with L1 HW validation
Definition at line 1460 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_L1Reco | ( | self, | |
sequence = "L1Reco" |
|||
| ) |
Enrich the schedule with L1 reconstruction
Definition at line 1467 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_L1REPACK | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection
Definition at line 1363 of file ConfigBuilder.py.
01364 : 01365 """ Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection""" 01366 if sequence is not 'GT': 01367 print 'Running the full L1 emulator is not supported yet' 01368 raise Exception('unsupported feature') 01369 if sequence is 'GT': 01370 self.loadAndRemember('Configuration/StandardSequences/SimL1EmulatorRepack_GT_cff') 01371 if self._options.scenario == 'HeavyIons': 01372 self.renameInputTagsInSequence("SimL1Emulator","rawDataCollector","rawDataRepacker") 01373 self.scheduleSequence('SimL1Emulator','L1simulation_step') 01374
| def ConfigBuilder::ConfigBuilder::prepare_LHE | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1205 of file ConfigBuilder.py.
01206 : 01207 #load the fragment 01208 ##make it loadable 01209 loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','').replace('/','.') 01210 print "Loading lhe fragment from",loadFragment 01211 __import__(loadFragment) 01212 self.process.load(loadFragment) 01213 ##inline the modules 01214 self._options.inlineObjets+=','+sequence 01215 01216 getattr(self.process,sequence).nEvents = int(self._options.number) 01217 01218 #schedule it 01219 self.process.lhe_step = cms.Path( getattr( self.process,sequence) ) 01220 self.schedule.append( self.process.lhe_step )
| def ConfigBuilder::ConfigBuilder::prepare_POSTRECO | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the postreco step
Definition at line 1561 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RAW2DIGI | ( | self, | |
sequence = "RawToDigi" |
|||
| ) |
Definition at line 1453 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RAW2RECO | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1442 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RECO | ( | self, | |
sequence = "reconstruction" |
|||
| ) |
Enrich the schedule with reconstruction
Definition at line 1505 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_REPACK | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1345 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_SIM | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the simulation step
Definition at line 1294 of file ConfigBuilder.py.
01295 : 01296 """ Enrich the schedule with the simulation step""" 01297 self.loadAndRemember(self.SIMDefaultCFF) 01298 if self._options.gflash==True: 01299 self.loadAndRemember("Configuration/StandardSequences/GFlashSIM_cff") 01300 01301 if self._options.magField=='0T': 01302 self.executeAndRemember("process.g4SimHits.UseMagneticField = cms.bool(False)") 01303 01304 if self._options.himix==True: 01305 if self._options.geometry in defaultOptions.geometryExtendedOptions: 01306 self.loadAndRemember("SimGeneral/MixingModule/himixSIMExtended_cff") 01307 else: 01308 self.loadAndRemember("SimGeneral/MixingModule/himixSIMIdeal_cff") 01309 01310 self.scheduleSequence('psim','simulation_step') 01311 return
| def ConfigBuilder::ConfigBuilder::prepare_SKIM | ( | self, | |
sequence = "all" |
|||
| ) |
Enrich the schedule with skimming fragments
Definition at line 1511 of file ConfigBuilder.py.
01512 : 01513 ''' Enrich the schedule with skimming fragments''' 01514 skimConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.SKIMDefaultCFF) 01515 sequence = sequence.split('.')[-1] 01516 01517 skimlist=sequence.split('+') 01518 ## support @Mu+DiJet+@Electron configuration via autoSkim.py 01519 from Configuration.Skimming.autoSkim import autoSkim 01520 self.expandMapping(skimlist,autoSkim) 01521 01522 #print "dictionnary for skims:",skimConfig.__dict__ 01523 for skim in skimConfig.__dict__: 01524 skimstream = getattr(skimConfig,skim) 01525 if isinstance(skimstream,cms.Path): 01526 #black list the alca path so that they do not appear in the cfg 01527 self.blacklist_paths.append(skimstream) 01528 if (not isinstance(skimstream,cms.FilteredStream)): 01529 continue 01530 shortname = skim.replace('SKIMStream','') 01531 if (sequence=="all"): 01532 self.addExtraStream(skim,skimstream) 01533 elif (shortname in skimlist): 01534 self.addExtraStream(skim,skimstream) 01535 #add a DQM eventcontent for this guy 01536 if self._options.datatier!="": 01537 self.process.load(self.EVTCONTDefaultCFF) 01538 skimstreamDQM = cms.FilteredStream( 01539 responsible = skimstream.responsible, 01540 name = skimstream.name+'DQM', 01541 paths = skimstream.paths, 01542 selectEvents = skimstream.selectEvents, 01543 content = self._options.datatier+'EventContent', 01544 dataTier = cms.untracked.string(self._options.datatier) 01545 ) 01546 self.addExtraStream(skim+'DQM',skimstreamDQM) 01547 for i in range(skimlist.count(shortname)): 01548 skimlist.remove(shortname) 01549 01550 01551 01552 if (skimlist.__len__()!=0 and sequence!="all"): 01553 print 'WARNING, possible typo with SKIM:'+'+'.join(skimlist) 01554 raise Exception('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
| def ConfigBuilder::ConfigBuilder::prepare_USER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with a user defined sequence
Definition at line 1555 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_VALIDATION | ( | self, | |
sequence = 'validation' |
|||
| ) |
Definition at line 1568 of file ConfigBuilder.py.
01569 : 01570 self.loadDefaultOrSpecifiedCFF(sequence,self.VALIDATIONDefaultCFF) 01571 #in case VALIDATION:something:somethingelse -> something,somethingelse 01572 sequence=sequence.split('.')[-1] 01573 if sequence.find(',')!=-1: 01574 prevalSeqName=sequence.split(',')[0] 01575 valSeqName=sequence.split(',')[1] 01576 else: 01577 postfix='' 01578 if sequence: 01579 postfix='_'+sequence 01580 prevalSeqName='prevalidation'+postfix 01581 valSeqName='validation'+postfix 01582 if not hasattr(self.process,valSeqName): 01583 prevalSeqName='' 01584 valSeqName=sequence 01585 01586 if not 'DIGI' in self.stepMap and not 'FASTSIM' in self.stepMap and not valSeqName.startswith('genvalid'): 01587 if self._options.restoreRNDSeeds==False and not self._options.restoreRNDSeeds==True: 01588 self._options.restoreRNDSeeds=True 01589 01590 #rename the HLT process in validation steps 01591 if ('HLT' in self.stepMap and not 'FASTSIM' in self.stepMap) or self._options.hltProcess: 01592 self.renameHLTprocessInSequence(valSeqName) 01593 if prevalSeqName: 01594 self.renameHLTprocessInSequence(prevalSeqName) 01595 01596 if prevalSeqName: 01597 self.process.prevalidation_step = cms.Path( getattr(self.process, prevalSeqName ) ) 01598 self.schedule.append(self.process.prevalidation_step) 01599 01600 self.process.validation_step = cms.EndPath( getattr(self.process,valSeqName ) ) 01601 self.schedule.append(self.process.validation_step) 01602 01603 if not 'DIGI' in self.stepMap and not 'FASTSIM' in self.stepMap: 01604 self.executeAndRemember("process.mix.playback = True") 01605 01606 if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions): 01607 #will get in the schedule, smoothly 01608 self.process.validation_step._seq = self.process.genstepfilter * self.process.validation_step._seq 01609 01610 return 01611
| def ConfigBuilder::ConfigBuilder::profileOptions | ( | self | ) |
addIgProfService Function to add the igprof profile service so that you can dump in the middle of the run.
Definition at line 221 of file ConfigBuilder.py.
00222 : 00223 """ 00224 addIgProfService 00225 Function to add the igprof profile service so that you can dump in the middle 00226 of the run. 00227 """ 00228 profileOpts = self._options.profile.split(':') 00229 profilerStart = 1 00230 profilerInterval = 100 00231 profilerFormat = None 00232 profilerJobFormat = None 00233 00234 if len(profileOpts): 00235 #type, given as first argument is unused here 00236 profileOpts.pop(0) 00237 if len(profileOpts): 00238 startEvent = profileOpts.pop(0) 00239 if not startEvent.isdigit(): 00240 raise Exception("%s is not a number" % startEvent) 00241 profilerStart = int(startEvent) 00242 if len(profileOpts): 00243 eventInterval = profileOpts.pop(0) 00244 if not eventInterval.isdigit(): 00245 raise Exception("%s is not a number" % eventInterval) 00246 profilerInterval = int(eventInterval) 00247 if len(profileOpts): 00248 profilerFormat = profileOpts.pop(0) 00249 00250 00251 if not profilerFormat: 00252 profilerFormat = "%s___%s___%s___%s___%s___%s___%%I.gz" % (self._options.evt_type.replace("_cfi", ""), 00253 self._options.step, 00254 self._options.pileup, 00255 self._options.conditions, 00256 self._options.datatier, 00257 self._options.profileTypeLabel) 00258 if not profilerJobFormat and profilerFormat.endswith(".gz"): 00259 profilerJobFormat = profilerFormat.replace(".gz", "_EndOfJob.gz") 00260 elif not profilerJobFormat: 00261 profilerJobFormat = profilerFormat + "_EndOfJob.gz" 00262 00263 return (profilerStart,profilerInterval,profilerFormat,profilerJobFormat)
| def ConfigBuilder::ConfigBuilder::renameHLTprocessInSequence | ( | self, | |
| sequence, | |||
proc = None, |
|||
HLTprocess = 'HLT' |
|||
| ) |
Definition at line 1681 of file ConfigBuilder.py.
01682 : 01683 if self._options.hltProcess: 01684 proc=self._options.hltProcess 01685 else: 01686 proc=self.process.name_() 01687 if proc==HLTprocess: return 01688 # look up all module in dqm sequence 01689 print "replacing %s process name - sequence %s will use '%s'" % (HLTprocess,sequence, proc) 01690 getattr(self.process,sequence).visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor(HLTprocess,proc,whitelist = ("subSystemFolder",))) 01691 if 'from Configuration.PyReleaseValidation.ConfigBuilder import ConfigBuilder' not in self.additionalCommands: 01692 self.additionalCommands.append('from Configuration.PyReleaseValidation.ConfigBuilder import ConfigBuilder') 01693 self.additionalCommands.append('process.%s.visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor("%s", "%s", whitelist = ("subSystemFolder",)))'% (sequence,HLTprocess, proc)) 01694
| def ConfigBuilder::ConfigBuilder::renameInputTagsInSequence | ( | self, | |
| sequence, | |||
oldT = "rawDataCollector", |
|||
newT = "rawDataRepacker" |
|||
| ) |
Definition at line 1671 of file ConfigBuilder.py.
01672 : 01673 print "Replacing all InputTag %s => %s"%(oldT,newT) 01674 from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag 01675 massSearchReplaceAnyInputTag(getattr(self.process,sequence),oldT,newT) 01676 loadMe='from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag' 01677 if not loadMe in self.additionalCommands: 01678 self.additionalCommands.append(loadMe) 01679 self.additionalCommands.append('massSearchReplaceAnyInputTag(process.%s,"%s","%s",False,True)'%(sequence,oldT,newT))
| def ConfigBuilder::ConfigBuilder::scheduleSequence | ( | self, | |
| seq, | |||
| prefix, | |||
what = 'Path' |
|||
| ) |
Definition at line 1112 of file ConfigBuilder.py.
01113 : 01114 if not what in ['Path','EndPath']: 01115 raise Exception("Can only schedule a Path or and EndPath."+str(what)+"is not allowed") 01116 if '*' in seq: 01117 #create only one path with all sequences in it 01118 for i,s in enumerate(seq.split('*')): 01119 if i==0: 01120 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, s) )) 01121 else: 01122 p=getattr(self.process,prefix) 01123 p+=getattr(self.process, s) 01124 self.schedule.append(getattr(self.process,prefix)) 01125 return 01126 else: 01127 #create as many path as many sequences 01128 if not '+' in seq: 01129 if self.nextScheduleIsConditional and what=='Path': 01130 self.conditionalPaths.append(prefix) 01131 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, seq) )) 01132 self.schedule.append(getattr(self.process,prefix)) 01133 else: 01134 for i,s in enumerate(seq.split('+')): 01135 sn=prefix+'%d'%(i) 01136 setattr(self.process,sn,getattr(cms,what)( getattr(self.process, s) )) 01137 self.schedule.append(getattr(self.process,sn)) 01138 return
| def ConfigBuilder::ConfigBuilder::scheduleSequenceAtEnd | ( | self, | |
| seq, | |||
| prefix | |||
| ) |
Definition at line 1139 of file ConfigBuilder.py.
ConfigBuilder::ConfigBuilder::_options [private] |
Definition at line 156 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 1747 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
load the relevant part
needs to be in reverse order
print it in the configuration
Definition at line 1475 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
put the filtering path in the schedule
Definition at line 156 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
put it before all the other paths
Definition at line 156 of file ConfigBuilder.py.
adding standard sequences might change the inputEventContent option and therefore needs to be finalized after
Definition at line 1870 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 318 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 811 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
Definition at line 156 of file ConfigBuilder.py.
1.7.3