|
|
|
The main building routines
Definition at line 151 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::__init__ | ( | self, | |
| options, | |||
process = None, |
|||
with_output = False, |
|||
with_input = False |
|||
| ) |
options taken from old cmsDriver and optparse
Definition at line 154 of file ConfigBuilder.py.
00155 : 00156 """options taken from old cmsDriver and optparse """ 00157 00158 options.outfile_name = options.dirout+options.fileout 00159 00160 self._options = options 00161 00162 if self._options.isData and options.isMC: 00163 raise Exception("ERROR: You may specify only --data or --mc, not both") 00164 #if not self._options.conditions: 00165 # raise Exception("ERROR: No conditions given!\nPlease specify conditions. E.g. via --conditions=IDEAL_30X::All") 00166 00167 if hasattr(self._options,"datatier") and self._options.datatier and 'DQMROOT' in self._options.datatier and 'ENDJOB' in self._options.step: 00168 self._options.step=self._options.step.replace(',ENDJOB','') 00169 00170 # what steps are provided by this class? 00171 stepList = [re.sub(r'^prepare_', '', methodName) for methodName in ConfigBuilder.__dict__ if methodName.startswith('prepare_')] 00172 self.stepMap={} 00173 self.stepKeys=[] 00174 for step in self._options.step.split(","): 00175 if step=='': continue 00176 stepParts = step.split(":") 00177 stepName = stepParts[0] 00178 if stepName not in stepList and not stepName.startswith('re'): 00179 raise ValueError("Step "+stepName+" unknown") 00180 if len(stepParts)==1: 00181 self.stepMap[stepName]="" 00182 elif len(stepParts)==2: 00183 self.stepMap[stepName]=stepParts[1].split('+') 00184 elif len(stepParts)==3: 00185 self.stepMap[stepName]=(stepParts[2].split('+'),stepParts[1]) 00186 else: 00187 raise ValueError("Step definition "+step+" invalid") 00188 self.stepKeys.append(stepName) 00189 00190 #print "map of steps is:",self.stepMap 00191 00192 self.with_output = with_output 00193 if hasattr(self._options,"no_output_flag") and self._options.no_output_flag: 00194 self.with_output = False 00195 self.with_input = with_input 00196 if process == None: 00197 self.process = cms.Process(self._options.name) 00198 else: 00199 self.process = process 00200 self.imports = [] 00201 self.define_Configs() 00202 self.schedule = list() 00203 00204 # we are doing three things here: 00205 # creating a process to catch errors 00206 # building the code to re-create the process 00207 00208 00209 00210 self.additionalCommands = [] 00211 # TODO: maybe a list of to be dumped objects would help as well 00212 self.blacklist_paths = [] 00213 self.addedObjects = [] 00214 self.additionalOutputs = {} 00215 00216 self.productionFilterSequence = None 00217 self.nextScheduleIsConditional=False 00218 self.conditionalPaths=[]
| def ConfigBuilder::ConfigBuilder::addCommon | ( | self | ) |
Definition at line 283 of file ConfigBuilder.py.
00284 : 00285 if 'HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys(): 00286 self.process.options = cms.untracked.PSet( Rethrow = cms.untracked.vstring('ProductNotFound'),fileMode = cms.untracked.string('FULLMERGE')) 00287 else: 00288 self.process.options = cms.untracked.PSet( ) 00289 self.addedObjects.append(("","options")) 00290 00291 if self._options.lazy_download: 00292 self.process.AdaptorConfig = cms.Service("AdaptorConfig", 00293 stats = cms.untracked.bool(True), 00294 enable = cms.untracked.bool(True), 00295 cacheHint = cms.untracked.string("lazy-download"), 00296 readHint = cms.untracked.string("read-ahead-buffered") 00297 ) 00298 self.addedObjects.append(("Setup lazy download","AdaptorConfig")) 00299 00300 #self.process.cmsDriverCommand = cms.untracked.PSet( command=cms.untracked.string('cmsDriver.py '+self._options.arguments) ) 00301 #self.addedObjects.append(("what cmsDriver command was used","cmsDriverCommand")) 00302 00303 if self._options.profile: 00304 (start, interval, eventFormat, jobFormat)=self.profileOptions() 00305 self.process.IgProfService = cms.Service("IgProfService", 00306 reportFirstEvent = cms.untracked.int32(start), 00307 reportEventInterval = cms.untracked.int32(interval), 00308 reportToFileAtPostEvent = cms.untracked.string("| gzip -c > %s"%(eventFormat)), 00309 reportToFileAtPostEndJob = cms.untracked.string("| gzip -c > %s"%(jobFormat))) 00310 self.addedObjects.append(("Setup IGProf Service for profiling","IgProfService"))
| def ConfigBuilder::ConfigBuilder::addConditions | ( | self | ) |
Add conditions to the process
Definition at line 711 of file ConfigBuilder.py.
00712 : 00713 """Add conditions to the process""" 00714 if not self._options.conditions: return 00715 00716 if 'FrontierConditions_GlobalTag' in self._options.conditions: 00717 print 'using FrontierConditions_GlobalTag in --conditions is not necessary anymore and will be deprecated soon. please update your command line' 00718 self._options.conditions = self._options.conditions.replace("FrontierConditions_GlobalTag,",'') 00719 00720 self.loadAndRemember(self.ConditionsDefaultCFF) 00721 00722 from Configuration.AlCa.GlobalTag import GlobalTag 00723 self.process.GlobalTag = GlobalTag(self.process.GlobalTag, self._options.conditions, self._options.custom_conditions) 00724 self.additionalCommands.append('from Configuration.AlCa.GlobalTag import GlobalTag') 00725 self.additionalCommands.append('process.GlobalTag = GlobalTag(process.GlobalTag, %s, %s)' % (repr(self._options.conditions), repr(self._options.custom_conditions))) 00726 00727 if self._options.slhc: 00728 self.loadAndRemember("SLHCUpgradeSimulations/Geometry/fakeConditions_%s_cff"%(self._options.slhc,)) 00729
| def ConfigBuilder::ConfigBuilder::addCustomise | ( | self | ) |
Include the customise code
Definition at line 730 of file ConfigBuilder.py.
00731 : 00732 """Include the customise code """ 00733 00734 custOpt=self._options.customisation_file.split(",") 00735 custMap={} 00736 for opt in custOpt: 00737 if opt=='': continue 00738 if opt.count('.')>1: 00739 raise Exception("more than . in the specification:"+opt) 00740 fileName=opt.split('.')[0] 00741 if opt.count('.')==0: rest='customise' 00742 else: 00743 rest=opt.split('.')[1] 00744 if rest=='py': rest='customise' #catch the case of --customise file.py 00745 00746 if fileName in custMap: 00747 custMap[fileName].extend(rest.split('+')) 00748 else: 00749 custMap[fileName]=rest.split('+') 00750 00751 if len(custMap)==0: 00752 final_snippet='\n' 00753 else: 00754 final_snippet='\n# customisation of the process.\n' 00755 00756 allFcn=[] 00757 for opt in custMap: 00758 allFcn.extend(custMap[opt]) 00759 for fcn in allFcn: 00760 if allFcn.count(fcn)!=1: 00761 raise Exception("cannot specify twice "+fcn+" as a customisation method") 00762 00763 for f in custMap: 00764 # let python search for that package and do syntax checking at the same time 00765 packageName = f.replace(".py","").replace("/",".") 00766 __import__(packageName) 00767 package = sys.modules[packageName] 00768 00769 # now ask the package for its definition and pick .py instead of .pyc 00770 customiseFile = re.sub(r'\.pyc$', '.py', package.__file__) 00771 00772 final_snippet+='\n# Automatic addition of the customisation function from '+packageName+'\n' 00773 if self._options.inline_custom: 00774 for line in file(customiseFile,'r'): 00775 if "import FWCore.ParameterSet.Config" in line: 00776 continue 00777 final_snippet += line 00778 else: 00779 final_snippet += 'from %s import %s \n'%(packageName,','.join(custMap[f])) 00780 for fcn in custMap[f]: 00781 print "customising the process with",fcn,"from",f 00782 if not hasattr(package,fcn): 00783 #bound to fail at run time 00784 raise Exception("config "+f+" has no function "+fcn) 00785 #execute the command 00786 self.process=getattr(package,fcn)(self.process) 00787 #and print it in the configuration 00788 final_snippet += "\n#call to customisation function "+fcn+" imported from "+packageName 00789 final_snippet += "\nprocess = %s(process)\n"%(fcn,) 00790 00791 if len(custMap)!=0: 00792 final_snippet += '\n# End of customisation functions\n' 00793 00794 ### now for a usuful command 00795 if self._options.customise_commands: 00796 import string 00797 final_snippet +='\n# Customisation from command line' 00798 for com in self._options.customise_commands.split('\\n'): 00799 com=string.lstrip(com) 00800 self.executeAndRemember(com) 00801 final_snippet +='\n'+com 00802 00803 return final_snippet
| def ConfigBuilder::ConfigBuilder::addExtraStream | ( | self, | |
| name, | |||
| stream, | |||
workflow = 'full' |
|||
| ) |
Definition at line 1032 of file ConfigBuilder.py.
01033 : 01034 # define output module and go from there 01035 output = cms.OutputModule("PoolOutputModule") 01036 if stream.selectEvents.parameters_().__len__()!=0: 01037 output.SelectEvents = stream.selectEvents 01038 else: 01039 output.SelectEvents = cms.untracked.PSet() 01040 output.SelectEvents.SelectEvents=cms.vstring() 01041 if isinstance(stream.paths,tuple): 01042 for path in stream.paths: 01043 output.SelectEvents.SelectEvents.append(path.label()) 01044 else: 01045 output.SelectEvents.SelectEvents.append(stream.paths.label()) 01046 01047 01048 01049 if isinstance(stream.content,str): 01050 evtPset=getattr(self.process,stream.content) 01051 for p in evtPset.parameters_(): 01052 setattr(output,p,getattr(evtPset,p)) 01053 if not self._options.inlineEventContent: 01054 def doNotInlineEventContent(instance,label = "process."+stream.content+".outputCommands"): 01055 return label 01056 output.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 01057 else: 01058 output.outputCommands = stream.content 01059 01060 01061 output.fileName = cms.untracked.string(self._options.dirout+stream.name+'.root') 01062 01063 output.dataset = cms.untracked.PSet( dataTier = stream.dataTier, 01064 filterName = cms.untracked.string(stream.name)) 01065 01066 if self._options.filtername: 01067 output.dataset.filterName= cms.untracked.string(self._options.filtername+"_"+stream.name) 01068 01069 #add an automatic flushing to limit memory consumption 01070 output.eventAutoFlushCompressedSize=cms.untracked.int32(5*1024*1024) 01071 01072 if workflow in ("producers,full"): 01073 if isinstance(stream.paths,tuple): 01074 for path in stream.paths: 01075 self.schedule.append(path) 01076 else: 01077 self.schedule.append(stream.paths) 01078 01079 01080 # in case of relvals we don't want to have additional outputs 01081 if (not self._options.relval) and workflow in ("full","output"): 01082 self.additionalOutputs[name] = output 01083 setattr(self.process,name,output) 01084 01085 if workflow == 'output': 01086 # adjust the select events to the proper trigger results from previous process 01087 filterList = output.SelectEvents.SelectEvents 01088 for i, filter in enumerate(filterList): 01089 filterList[i] = filter+":"+self._options.triggerResultsProcess 01090 01091 return output
| def ConfigBuilder::ConfigBuilder::addMaxEvents | ( | self | ) |
Here we decide how many evts will be processed
Definition at line 311 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::addOutput | ( | self | ) |
Add output module to the process
Definition at line 429 of file ConfigBuilder.py.
00430 : 00431 """ Add output module to the process """ 00432 result="" 00433 if self._options.outputDefinition: 00434 if self._options.datatier: 00435 print "--datatier & --eventcontent options ignored" 00436 00437 def anyOf(listOfKeys,dict,opt=None): 00438 for k in listOfKeys: 00439 if k in dict: 00440 toReturn=dict[k] 00441 dict.pop(k) 00442 return toReturn 00443 if opt!=None: 00444 return opt 00445 else: 00446 raise Exception("any of "+','.join(listOfKeys)+" are mandatory entries of --output options") 00447 00448 #new output convention with a list of dict 00449 outList = eval(self._options.outputDefinition) 00450 for (id,outDefDict) in enumerate(outList): 00451 outDefDictStr=outDefDict.__str__() 00452 if not isinstance(outDefDict,dict): 00453 raise Exception("--output needs to be passed a list of dict"+self._options.outputDefinition+" is invalid") 00454 #requires option: tier 00455 theTier=anyOf(['t','tier','dataTier'],outDefDict) 00456 #optional option: eventcontent, filtername, selectEvents, moduleLabel, filename 00457 ## event content 00458 theStreamType=anyOf(['e','ec','eventContent','streamType'],outDefDict,theTier) 00459 theFilterName=anyOf(['f','ftN','filterName'],outDefDict,'') 00460 theSelectEvent=anyOf(['s','sE','selectEvents'],outDefDict,'') 00461 theModuleLabel=anyOf(['l','mL','moduleLabel'],outDefDict,'') 00462 theExtraOutputCommands=anyOf(['o','oC','outputCommands'],outDefDict,'') 00463 # module label has a particular role 00464 if not theModuleLabel: 00465 tryNames=[theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+'output', 00466 theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+'output', 00467 theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+theSelectEvent.split(',')[0].replace(':','for').replace(' ','')+'output' 00468 ] 00469 for name in tryNames: 00470 if not hasattr(self.process,name): 00471 theModuleLabel=name 00472 break 00473 if not theModuleLabel: 00474 raise Exception("cannot find a module label for specification: "+outDefDictStr) 00475 if id==0: 00476 defaultFileName=self._options.outfile_name 00477 else: 00478 defaultFileName=self._options.outfile_name.replace('.root','_in'+theTier+'.root') 00479 00480 theFileName=self._options.dirout+anyOf(['fn','fileName'],outDefDict,defaultFileName) 00481 if not theFileName.endswith('.root'): 00482 theFileName+='.root' 00483 00484 if len(outDefDict.keys()): 00485 raise Exception("unused keys from --output options: "+','.join(outDefDict.keys())) 00486 if theStreamType=='DQMROOT': theStreamType='DQM' 00487 if theStreamType=='ALL': 00488 theEventContent = cms.PSet(outputCommands = cms.untracked.vstring('keep *')) 00489 else: 00490 theEventContent = getattr(self.process, theStreamType+"EventContent") 00491 00492 if theStreamType=='ALCARECO' and not theFilterName: 00493 theFilterName='StreamALCACombined' 00494 00495 CppType='PoolOutputModule' 00496 if theStreamType=='DQM' and theTier=='DQMROOT': CppType='DQMRootOutputModule' 00497 output = cms.OutputModule(CppType, 00498 theEventContent.clone(), 00499 fileName = cms.untracked.string(theFileName), 00500 dataset = cms.untracked.PSet( 00501 dataTier = cms.untracked.string(theTier), 00502 filterName = cms.untracked.string(theFilterName)) 00503 ) 00504 if not theSelectEvent and hasattr(self.process,'generation_step'): 00505 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step')) 00506 if not theSelectEvent and hasattr(self.process,'filtering_step'): 00507 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step')) 00508 if theSelectEvent: 00509 output.SelectEvents =cms.untracked.PSet(SelectEvents = cms.vstring(theSelectEvent)) 00510 00511 if hasattr(self.process,theModuleLabel): 00512 raise Exception("the current process already has a module "+theModuleLabel+" defined") 00513 #print "creating output module ",theModuleLabel 00514 setattr(self.process,theModuleLabel,output) 00515 outputModule=getattr(self.process,theModuleLabel) 00516 setattr(self.process,theModuleLabel+'_step',cms.EndPath(outputModule)) 00517 path=getattr(self.process,theModuleLabel+'_step') 00518 self.schedule.append(path) 00519 00520 if not self._options.inlineEventContent and hasattr(self.process,theStreamType+"EventContent"): 00521 def doNotInlineEventContent(instance,label = "cms.untracked.vstring(process."+theStreamType+"EventContent.outputCommands)"): 00522 return label 00523 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 00524 if theExtraOutputCommands: 00525 if not isinstance(theExtraOutputCommands,list): 00526 raise Exception("extra ouput command in --option must be a list of strings") 00527 if hasattr(self.process,theStreamType+"EventContent"): 00528 self.executeAndRemember('process.%s.outputCommands.extend(%s)'%(theModuleLabel,theExtraOutputCommands)) 00529 else: 00530 outputModule.outputCommands.extend(theExtraOutputCommands) 00531 00532 result+="\nprocess."+theModuleLabel+" = "+outputModule.dumpPython() 00533 00534 ##ends the --output options model 00535 return result 00536 00537 streamTypes=self._options.eventcontent.split(',') 00538 tiers=self._options.datatier.split(',') 00539 if not self._options.outputDefinition and len(streamTypes)!=len(tiers): 00540 raise Exception("number of event content arguments does not match number of datatier arguments") 00541 00542 # if the only step is alca we don't need to put in an output 00543 if self._options.step.split(',')[0].split(':')[0] == 'ALCA': 00544 return "\n" 00545 00546 for i,(streamType,tier) in enumerate(zip(streamTypes,tiers)): 00547 if streamType=='': continue 00548 if streamType=='DQMROOT': streamType='DQM' 00549 theEventContent = getattr(self.process, streamType+"EventContent") 00550 if i==0: 00551 theFileName=self._options.outfile_name 00552 theFilterName=self._options.filtername 00553 else: 00554 theFileName=self._options.outfile_name.replace('.root','_in'+streamType+'.root') 00555 theFilterName=self._options.filtername 00556 CppType='PoolOutputModule' 00557 if streamType=='DQM' and tier=='DQMROOT': CppType='DQMRootOutputModule' 00558 output = cms.OutputModule(CppType, 00559 theEventContent, 00560 fileName = cms.untracked.string(theFileName), 00561 dataset = cms.untracked.PSet(dataTier = cms.untracked.string(tier), 00562 filterName = cms.untracked.string(theFilterName) 00563 ) 00564 ) 00565 if hasattr(self.process,"generation_step"): 00566 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step')) 00567 if hasattr(self.process,"filtering_step"): 00568 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step')) 00569 00570 if streamType=='ALCARECO': 00571 output.dataset.filterName = cms.untracked.string('StreamALCACombined') 00572 00573 outputModuleName=streamType+'output' 00574 setattr(self.process,outputModuleName,output) 00575 outputModule=getattr(self.process,outputModuleName) 00576 setattr(self.process,outputModuleName+'_step',cms.EndPath(outputModule)) 00577 path=getattr(self.process,outputModuleName+'_step') 00578 self.schedule.append(path) 00579 00580 if not self._options.inlineEventContent: 00581 def doNotInlineEventContent(instance,label = "process."+streamType+"EventContent.outputCommands"): 00582 return label 00583 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 00584 00585 result+="\nprocess."+outputModuleName+" = "+outputModule.dumpPython() 00586 00587 return result
| def ConfigBuilder::ConfigBuilder::addSource | ( | self | ) |
Here the source is built. Priority: file, generator
Definition at line 316 of file ConfigBuilder.py.
00317 : 00318 """Here the source is built. Priority: file, generator""" 00319 self.addedObjects.append(("Input source","source")) 00320 00321 def filesFromOption(self): 00322 for entry in self._options.filein.split(','): 00323 print "entry",entry 00324 if entry.startswith("filelist:"): 00325 filesFromList(entry[9:],self.process.source) 00326 elif entry.startswith("dbs:"): 00327 filesFromDBSQuery('find file where dataset = %s'%(entry[4:]),self.process.source) 00328 else: 00329 self.process.source.fileNames.append(self._options.dirin+entry) 00330 if self._options.secondfilein: 00331 if not hasattr(self.process.source,"secondaryFileNames"): 00332 raise Exception("--secondfilein not compatible with "+self._options.filetype+"input type") 00333 for entry in self._options.secondfilein.split(','): 00334 print "entry",entry 00335 if entry.startswith("filelist:"): 00336 self.process.source.secondaryFileNames.extend((filesFromList(entry[9:]))[0]) 00337 elif entry.startswith("dbs:"): 00338 self.process.source.secondaryFileNames.extend((filesFromDBSQuery('find file where dataset = %s'%(entry[4:])))[0]) 00339 else: 00340 self.process.source.secondaryFileNames.append(self._options.dirin+entry) 00341 00342 if self._options.filein or self._options.dbsquery: 00343 if self._options.filetype == "EDM": 00344 self.process.source=cms.Source("PoolSource", 00345 fileNames = cms.untracked.vstring(), 00346 secondaryFileNames= cms.untracked.vstring()) 00347 filesFromOption(self) 00348 elif self._options.filetype == "DAT": 00349 self.process.source=cms.Source("NewEventStreamFileReader",fileNames = cms.untracked.vstring()) 00350 filesFromOption(self) 00351 elif self._options.filetype == "LHE": 00352 self.process.source=cms.Source("LHESource", fileNames = cms.untracked.vstring()) 00353 if self._options.filein.startswith("lhe:"): 00354 #list the article directory automatically 00355 args=self._options.filein.split(':') 00356 article=args[1] 00357 print 'LHE input from article ',article 00358 location='/store/lhe/' 00359 import os 00360 textOfFiles=os.popen('cmsLHEtoEOSManager.py -l '+article) 00361 for line in textOfFiles: 00362 for fileName in [x for x in line.split() if '.lhe' in x]: 00363 self.process.source.fileNames.append(location+article+'/'+fileName) 00364 if len(args)>2: 00365 self.process.source.skipEvents = cms.untracked.uint32(int(args[2])) 00366 else: 00367 filesFromOption(self) 00368 00369 00370 elif self._options.filetype == "DQM": 00371 self.process.source=cms.Source("DQMRootSource", 00372 fileNames = cms.untracked.vstring()) 00373 filesFromOption(self) 00374 00375 if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"): 00376 self.process.source.processingMode = cms.untracked.string("RunsAndLumis") 00377 00378 if self._options.dbsquery!='': 00379 self.process.source=cms.Source("PoolSource", fileNames = cms.untracked.vstring(),secondaryFileNames = cms.untracked.vstring()) 00380 filesFromDBSQuery(self._options.dbsquery,self.process.source) 00381 00382 if self._options.inputCommands: 00383 if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring() 00384 for command in self._options.inputCommands.split(','): 00385 # remove whitespace around the keep/drop statements 00386 command = command.strip() 00387 if command=='': continue 00388 self.process.source.inputCommands.append(command) 00389 if not self._options.dropDescendant: 00390 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False) 00391 00392 00393 00394 if self._options.lumiToProcess: 00395 import FWCore.PythonUtilities.LumiList as LumiList 00396 self.process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange( LumiList.LumiList(self._options.lumiToProcess).getCMSSWString().split(',') ) 00397 00398 if 'GEN' in self.stepMap or 'LHE' in self.stepMap or (not self._options.filein and hasattr(self._options, "evt_type")): 00399 if self.process.source is None: 00400 self.process.source=cms.Source("EmptySource") 00401 # if option himix is active, drop possibly duplicate DIGI-RAW info: 00402 if self._options.himix==True: 00403 self.process.source.inputCommands = cms.untracked.vstring('drop *','keep *_generator_*_*','keep *_g4SimHits_*_*') 00404 self.process.source.dropDescendantsOfDroppedBranches=cms.untracked.bool(False) 00405 00406 # modify source in case of run-dependent MC 00407 00408 self.runsAndWeights=None 00409 if self._options.runsAndWeightsForMC or self._options.runsScenarioForMC : 00410 if not self._options.isMC : 00411 raise Exception("options --runsAndWeightsForMC and --runsScenarioForMC are only valid for MC") 00412 if self._options.runsAndWeightsForMC: 00413 self.runsAndWeights = eval(self._options.runsAndWeightsForMC) 00414 else: 00415 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights 00416 if type(RunsAndWeights[self._options.runsScenarioForMC])==str: 00417 __import__(RunsAndWeights[self._options.runsScenarioForMC]) 00418 self.runsAndWeights = sys.modules[RunsAndWeights[self._options.runsScenarioForMC]].runProbabilityDistribution 00419 else: 00420 self.runsAndWeights = RunsAndWeights[self._options.runsScenarioForMC] 00421 00422 if self.runsAndWeights: 00423 import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun 00424 ThrowAndSetRandomRun.throwAndSetRandomRun(self.process.source,self.runsAndWeights) 00425 self.additionalCommands.append('import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun') 00426 self.additionalCommands.append('ThrowAndSetRandomRun.throwAndSetRandomRun(process.source,%s)'%(self.runsAndWeights)) 00427 00428 return
| def ConfigBuilder::ConfigBuilder::addStandardSequences | ( | self | ) |
Add selected standard sequences to the process
Definition at line 588 of file ConfigBuilder.py.
00589 : 00590 """ 00591 Add selected standard sequences to the process 00592 """ 00593 # load the pile up file 00594 if self._options.pileup: 00595 pileupSpec=self._options.pileup.split(',')[0] 00596 from Configuration.StandardSequences.Mixing import Mixing,defineMixing 00597 if not pileupSpec in Mixing and '.' not in pileupSpec and 'file:' not in pileupSpec: 00598 raise Exception(pileupSpec+' is not a know mixing scenario:\n available are: '+'\n'.join(Mixing.keys())) 00599 if '.' in pileupSpec: 00600 mixingDict={'file':pileupSpec} 00601 elif pileupSpec.startswith('file:'): 00602 mixingDict={'file':pileupSpec[5:]} 00603 else: 00604 import copy 00605 mixingDict=copy.copy(Mixing[pileupSpec]) 00606 if len(self._options.pileup.split(','))>1: 00607 mixingDict.update(eval(self._options.pileup[self._options.pileup.find(',')+1:])) 00608 if 'file:' in pileupSpec: 00609 #the file is local 00610 self.process.load(mixingDict['file']) 00611 print "inlining mixing module configuration" 00612 self._options.inlineObjets+=',mix' 00613 else: 00614 self.loadAndRemember(mixingDict['file']) 00615 00616 mixingDict.pop('file') 00617 if self._options.pileup_input: 00618 if self._options.pileup_input.startswith('dbs'): 00619 mixingDict['F']=filesFromDBSQuery('find file where dataset = %s'%(self._options.pileup_input[4:],))[0] 00620 else: 00621 mixingDict['F']=self._options.pileup_input.split(',') 00622 specialization=defineMixing(mixingDict,'FASTSIM' in self.stepMap) 00623 for command in specialization: 00624 self.executeAndRemember(command) 00625 if len(mixingDict)!=0: 00626 raise Exception('unused mixing specification: '+mixingDict.keys().__str__()) 00627 00628 00629 # load the geometry file 00630 try: 00631 if len(self.stepMap): 00632 self.loadAndRemember(self.GeometryCFF) 00633 if 'SIM' in self.stepMap or 'reSIM' in self.stepMap: 00634 self.loadAndRemember(self.SimGeometryCFF) 00635 if self.geometryDBLabel: 00636 self.executeAndRemember('process.XMLFromDBSource.label = cms.string("%s")'%(self.geometryDBLabel)) 00637 except ImportError: 00638 print "Geometry option",self._options.geometry,"unknown." 00639 raise 00640 00641 if len(self.stepMap): 00642 self.loadAndRemember(self.magFieldCFF) 00643 00644 00645 for stepName in self.stepKeys: 00646 stepSpec = self.stepMap[stepName] 00647 print "Step:", stepName,"Spec:",stepSpec 00648 if stepName.startswith('re'): 00649 ##add the corresponding input content 00650 if stepName[2:] not in self._options.donotDropOnInput: 00651 self._options.inputEventContent='%s,%s'%(stepName.upper(),self._options.inputEventContent) 00652 stepName=stepName[2:] 00653 if stepSpec=="": 00654 getattr(self,"prepare_"+stepName)(sequence = getattr(self,stepName+"DefaultSeq")) 00655 elif type(stepSpec)==list: 00656 getattr(self,"prepare_"+stepName)(sequence = '+'.join(stepSpec)) 00657 elif type(stepSpec)==tuple: 00658 getattr(self,"prepare_"+stepName)(sequence = ','.join([stepSpec[1],'+'.join(stepSpec[0])])) 00659 else: 00660 raise ValueError("Invalid step definition") 00661 00662 if self._options.restoreRNDSeeds!=False: 00663 #it is either True, or a process name 00664 if self._options.restoreRNDSeeds==True: 00665 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateLabel=cms.untracked.string("randomEngineStateProducer")') 00666 else: 00667 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateTag=cms.untracked.InputTag("randomEngineStateProducer","","%s")'%(self._options.restoreRNDSeeds)) 00668 if self._options.inputEventContent or self._options.inputCommands: 00669 if self._options.inputCommands: 00670 self._options.inputCommands+='keep *_randomEngineStateProducer_*_*,' 00671 else: 00672 self._options.inputCommands='keep *_randomEngineStateProducer_*_*,'
| def ConfigBuilder::ConfigBuilder::build_production_info | ( | self, | |
| evt_type, | |||
| evtnumber | |||
| ) |
Add useful info for the production.
Definition at line 1843 of file ConfigBuilder.py.
01844 : 01845 """ Add useful info for the production. """ 01846 self.process.configurationMetadata=cms.untracked.PSet\ 01847 (version=cms.untracked.string("$Revision: 1.381.2.27 $"), 01848 name=cms.untracked.string("PyReleaseValidation"), 01849 annotation=cms.untracked.string(evt_type+ " nevts:"+str(evtnumber)) 01850 ) 01851 01852 self.addedObjects.append(("Production Info","configurationMetadata")) 01853
| def ConfigBuilder::ConfigBuilder::completeInputCommand | ( | self | ) |
Definition at line 673 of file ConfigBuilder.py.
00674 : 00675 if self._options.inputEventContent: 00676 import copy 00677 def dropSecondDropStar(iec): 00678 #drop occurence of 'drop *' in the list 00679 count=0 00680 for item in iec: 00681 if item=='drop *': 00682 if count!=0: 00683 iec.remove(item) 00684 count+=1 00685 00686 00687 ## allow comma separated input eventcontent 00688 if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring() 00689 for evct in self._options.inputEventContent.split(','): 00690 if evct=='': continue 00691 theEventContent = getattr(self.process, evct+"EventContent") 00692 if hasattr(theEventContent,'outputCommands'): 00693 self.process.source.inputCommands.extend(copy.copy(theEventContent.outputCommands)) 00694 if hasattr(theEventContent,'inputCommands'): 00695 self.process.source.inputCommands.extend(copy.copy(theEventContent.inputCommands)) 00696 00697 dropSecondDropStar(self.process.source.inputCommands) 00698 00699 if not self._options.dropDescendant: 00700 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False) 00701 00702 #re-order the input commands to put drop first 00703 ## drop is before keep: so just sort 00704 self.process.source.inputCommands.sort() 00705 #remove that inconsistency: cannot be dropping * and keeping * 00706 if 'keep *' in self.process.source.inputCommands and 'drop *' in self.process.source.inputCommands: 00707 self.process.source.inputCommands.remove( 'keep *' ) 00708 00709 00710 return
| def ConfigBuilder::ConfigBuilder::define_Configs | ( | self | ) |
Definition at line 808 of file ConfigBuilder.py.
00809 : 00810 if len(self.stepMap): 00811 self.loadAndRemember('Configuration/StandardSequences/Services_cff') 00812 if self._options.particleTable not in defaultOptions.particleTableList: 00813 print 'Invalid particle table provided. Options are:' 00814 print defaultOptions.particleTable 00815 sys.exit(-1) 00816 else: 00817 if len(self.stepMap): 00818 self.loadAndRemember('SimGeneral.HepPDTESSource.'+self._options.particleTable+'_cfi') 00819 00820 self.loadAndRemember('FWCore/MessageService/MessageLogger_cfi') 00821 00822 self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreams_cff" 00823 self.GENDefaultCFF="Configuration/StandardSequences/Generator_cff" 00824 self.SIMDefaultCFF="Configuration/StandardSequences/Sim_cff" 00825 self.DIGIDefaultCFF="Configuration/StandardSequences/Digi_cff" 00826 self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRaw_cff" 00827 self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1Emulator_cff' 00828 self.L1MENUDefaultCFF="Configuration/StandardSequences/L1TriggerDefaultMenu_cff" 00829 self.HLTDefaultCFF="Configuration/StandardSequences/HLTtable_cff" 00830 self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_Data_cff" 00831 self.L1RecoDefaultCFF="Configuration/StandardSequences/L1Reco_cff" 00832 self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_Data_cff" 00833 self.SKIMDefaultCFF="Configuration/StandardSequences/Skims_cff" 00834 self.POSTRECODefaultCFF="Configuration/StandardSequences/PostRecoGenerator_cff" 00835 self.VALIDATIONDefaultCFF="Configuration/StandardSequences/Validation_cff" 00836 self.L1HwValDefaultCFF = "Configuration/StandardSequences/L1HwVal_cff" 00837 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOffline_cff" 00838 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/Harvesting_cff" 00839 self.ALCAHARVESTDefaultCFF="Configuration/StandardSequences/AlCaHarvesting_cff" 00840 self.ENDJOBDefaultCFF="Configuration/StandardSequences/EndOfProcess_cff" 00841 self.ConditionsDefaultCFF = "Configuration/StandardSequences/FrontierConditions_GlobalTag_cff" 00842 self.CFWRITERDefaultCFF = "Configuration/StandardSequences/CrossingFrameWriter_cff" 00843 self.REPACKDefaultCFF="Configuration/StandardSequences/DigiToRaw_Repack_cff" 00844 00845 if "DATAMIX" in self.stepMap.keys(): 00846 self.DATAMIXDefaultCFF="Configuration/StandardSequences/DataMixer"+self._options.datamix+"_cff" 00847 self.DIGIDefaultCFF="Configuration/StandardSequences/DigiDM_cff" 00848 self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRawDM_cff" 00849 self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1EmulatorDM_cff' 00850 00851 self.ALCADefaultSeq=None 00852 self.LHEDefaultSeq='externalLHEProducer' 00853 self.GENDefaultSeq='pgen' 00854 self.SIMDefaultSeq=None 00855 self.DIGIDefaultSeq='pdigi' 00856 self.DATAMIXDefaultSeq=None 00857 self.DIGI2RAWDefaultSeq='DigiToRaw' 00858 self.HLTDefaultSeq='GRun' 00859 self.L1DefaultSeq=None 00860 self.L1REPACKDefaultSeq='GT' 00861 self.HARVESTINGDefaultSeq=None 00862 self.ALCAHARVESTDefaultSeq=None 00863 self.CFWRITERDefaultSeq=None 00864 self.RAW2DIGIDefaultSeq='RawToDigi' 00865 self.L1RecoDefaultSeq='L1Reco' 00866 if 'RAW2DIGI' in self.stepMap and 'RECO' in self.stepMap: 00867 self.RECODefaultSeq='reconstruction' 00868 else: 00869 self.RECODefaultSeq='reconstruction_fromRECO' 00870 00871 self.POSTRECODefaultSeq=None 00872 self.L1HwValDefaultSeq='L1HwVal' 00873 self.DQMDefaultSeq='DQMOffline' 00874 self.FASTSIMDefaultSeq='all' 00875 self.VALIDATIONDefaultSeq='' 00876 self.PATLayer0DefaultSeq='all' 00877 self.ENDJOBDefaultSeq='endOfProcess' 00878 self.REPACKDefaultSeq='DigiToRawRepack' 00879 00880 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContent_cff" 00881 00882 # if fastsim switch event content 00883 if "FASTSIM" in self.stepMap.keys(): 00884 self.GENDefaultSeq='pgen_genonly' 00885 self.EVTCONTDefaultCFF = "FastSimulation/Configuration/EventContent_cff" 00886 self.VALIDATIONDefaultCFF = "FastSimulation.Configuration.Validation_cff" 00887 00888 if not self._options.beamspot: 00889 self._options.beamspot=VtxSmearedDefaultKey 00890 00891 # if its MC then change the raw2digi 00892 if self._options.isMC==True: 00893 self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_cff" 00894 self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_cff" 00895 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineMC_cff" 00896 self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreamsMC_cff" 00897 else: 00898 self._options.beamspot = None 00899 00900 #patch for gen, due to backward incompatibility 00901 if 'reGEN' in self.stepMap: 00902 self.GENDefaultSeq='fixGenInfo' 00903 00904 if self._options.scenario=='cosmics': 00905 self.DIGIDefaultCFF="Configuration/StandardSequences/DigiCosmics_cff" 00906 self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionCosmics_cff" 00907 self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsCosmics_cff" 00908 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentCosmics_cff" 00909 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmics_cff" 00910 if self._options.isMC==True: 00911 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmicsMC_cff" 00912 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingCosmics_cff" 00913 self.RECODefaultSeq='reconstructionCosmics' 00914 self.DQMDefaultSeq='DQMOfflineCosmics' 00915 00916 if self._options.himix: 00917 print "From the presence of the himix option, we have determined that this is heavy ions and will use '--scenario HeavyIons'." 00918 self._options.scenario='HeavyIons' 00919 00920 if self._options.scenario=='HeavyIons': 00921 if not self._options.beamspot: 00922 self._options.beamspot=VtxSmearedHIDefaultKey 00923 self.HLTDefaultSeq = 'HIon' 00924 if not self._options.himix: 00925 self.GENDefaultSeq='pgen_hi' 00926 else: 00927 self.GENDefaultSeq='pgen_himix' 00928 self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationHeavyIons_cff" 00929 self.VALIDATIONDefaultSeq='' 00930 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentHeavyIons_cff" 00931 self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionHeavyIons_cff" 00932 self.RECODefaultSeq='reconstructionHeavyIons' 00933 self.ALCADefaultCFF = "Configuration/StandardSequences/AlCaRecoStreamsHeavyIons_cff" 00934 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIons_cff" 00935 self.DQMDefaultSeq='DQMOfflineHeavyIons' 00936 self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsHeavyIons_cff" 00937 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingHeavyIons_cff" 00938 if self._options.isMC==True: 00939 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIonsMC_cff" 00940 00941 00942 self.RAW2RECODefaultSeq=','.join([self.RAW2DIGIDefaultSeq,self.RECODefaultSeq]) 00943 00944 self.USERDefaultSeq='user' 00945 self.USERDefaultCFF=None 00946 00947 # the magnetic field 00948 if self._options.isData: 00949 if self._options.magField==defaultOptions.magField: 00950 print "magnetic field option forced to: AutoFromDBCurrent" 00951 self._options.magField='AutoFromDBCurrent' 00952 self.magFieldCFF = 'Configuration/StandardSequences/MagneticField_'+self._options.magField.replace('.','')+'_cff' 00953 self.magFieldCFF = self.magFieldCFF.replace("__",'_') 00954 00955 # the geometry 00956 self.GeometryCFF='Configuration/StandardSequences/GeometryRecoDB_cff' 00957 self.geometryDBLabel=None 00958 simGeometry='' 00959 if 'FASTSIM' in self.stepMap: 00960 if 'start' in self._options.conditions.lower(): 00961 self.GeometryCFF='FastSimulation/Configuration/Geometries_START_cff' 00962 else: 00963 self.GeometryCFF='FastSimulation/Configuration/Geometries_MC_cff' 00964 else: 00965 def inGeometryKeys(opt): 00966 from Configuration.StandardSequences.GeometryConf import GeometryConf 00967 if opt in GeometryConf: 00968 return GeometryConf[opt] 00969 else: 00970 return opt 00971 00972 geoms=self._options.geometry.split(',') 00973 if len(geoms)==1: geoms=inGeometryKeys(geoms[0]).split(',') 00974 if len(geoms)==2: 00975 #may specify the reco geometry 00976 if '/' in geoms[1] or '_cff' in geoms[1]: 00977 self.GeometryCFF=geoms[1] 00978 else: 00979 self.GeometryCFF='Configuration/StandardSequences/Geometry'+geoms[1]+'_cff' 00980 00981 if (geoms[0].startswith('DB:')): 00982 self.SimGeometryCFF='Configuration/StandardSequences/GeometrySimDB_cff' 00983 self.geometryDBLabel=geoms[0][3:] 00984 print "with DB:" 00985 else: 00986 if '/' in geoms[0] or '_cff' in geoms[0]: 00987 self.SimGeometryCFF=geoms[0] 00988 else: 00989 simGeometry=geoms[0] 00990 if self._options.gflash==True: 00991 self.SimGeometryCFF='Configuration/StandardSequences/Geometry'+geoms[0]+'GFlash_cff' 00992 else: 00993 self.SimGeometryCFF='Configuration/StandardSequences/Geometry'+geoms[0]+'_cff' 00994 00995 # synchronize the geometry configuration and the FullSimulation sequence to be used 00996 if simGeometry not in defaultOptions.geometryExtendedOptions: 00997 self.SIMDefaultCFF="Configuration/StandardSequences/SimIdeal_cff" 00998 00999 if self._options.scenario=='nocoll' or self._options.scenario=='cosmics': 01000 print simGeometry 01001 self.SIMDefaultCFF="Configuration/StandardSequences/SimNOBEAM_cff" 01002 self._options.beamspot='NoSmear' 01003 01004 01005 # Mixing 01006 if self._options.pileup=='default': 01007 from Configuration.StandardSequences.Mixing import MixingDefaultKey,MixingFSDefaultKey 01008 if 'FASTSIM' in self.stepMap: 01009 self._options.pileup=MixingFSDefaultKey 01010 else: 01011 self._options.pileup=MixingDefaultKey 01012 01013 #not driven by a default cff anymore 01014 if self._options.isData: 01015 self._options.pileup=None 01016 if self._options.isMC==True and self._options.himix==False: 01017 if 'FASTSIM' in self.stepMap: 01018 self._options.pileup='FS_'+self._options.pileup 01019 elif self._options.isMC==True and self._options.himix==True: 01020 self._options.pileup='HiMix' 01021 01022 01023 if self._options.slhc: 01024 if 'stdgeom' not in self._options.slhc: 01025 self.SimGeometryCFF='SLHCUpgradeSimulations.Geometry.%s_cmsSimIdealGeometryXML_cff'%(self._options.slhc,) 01026 self.DIGIDefaultCFF='SLHCUpgradeSimulations/Geometry/Digi_%s_cff'%(self._options.slhc,) 01027 if self._options.pileup!=defaultOptions.pileup: 01028 self._options.pileup='SLHC_%s_%s'%(self._options.pileup,self._options.slhc) 01029 01030 self.REDIGIDefaultSeq=self.DIGIDefaultSeq
| def ConfigBuilder::ConfigBuilder::executeAndRemember | ( | self, | |
| command | |||
| ) |
helper routine to remember replace statements
Definition at line 276 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::expandMapping | ( | self, | |
| seqList, | |||
| mapping, | |||
index = None |
|||
| ) |
Definition at line 1680 of file ConfigBuilder.py.
01681 : 01682 maxLevel=20 01683 level=0 01684 while '@' in repr(seqList) and level<maxLevel: 01685 level+=1 01686 for specifiedCommand in seqList: 01687 if specifiedCommand.startswith('@'): 01688 location=specifiedCommand[1:] 01689 if not location in mapping: 01690 raise Exception("Impossible to map "+location+" from "+repr(mapping)) 01691 mappedTo=mapping[location] 01692 if index!=None: 01693 mappedTo=mappedTo[index] 01694 seqList.remove(specifiedCommand) 01695 seqList.extend(mappedTo.split('+')) 01696 break; 01697 if level==maxLevel: 01698 raise Exception("Could not fully expand "+repr(seqList)+" from "+repr(mapping))
| def ConfigBuilder::ConfigBuilder::finalizeFastSimHLT | ( | self | ) |
Definition at line 1794 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::load | ( | self, | |
| includeFile | |||
| ) |
Definition at line 262 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::loadAndRemember | ( | self, | |
| includeFile | |||
| ) |
helper routine to load am memorize imports
Definition at line 267 of file ConfigBuilder.py.
00268 : 00269 """helper routine to load am memorize imports""" 00270 # we could make the imports a on-the-fly data method of the process instance itself 00271 # not sure if the latter is a good idea 00272 includeFile = includeFile.replace('/','.') 00273 self.imports.append(includeFile) 00274 self.process.load(includeFile) 00275 return sys.modules[includeFile]
| def ConfigBuilder::ConfigBuilder::loadDefaultOrSpecifiedCFF | ( | self, | |
| sequence, | |||
| defaultCFF | |||
| ) |
Definition at line 1097 of file ConfigBuilder.py.
01098 : 01099 if ( len(sequence.split('.'))==1 ): 01100 l=self.loadAndRemember(defaultCFF) 01101 elif ( len(sequence.split('.'))==2 ): 01102 l=self.loadAndRemember(sequence.split('.')[0]) 01103 sequence=sequence.split('.')[1] 01104 else: 01105 print "sub sequence configuration must be of the form dir/subdir/cff.a+b+c or cff.a" 01106 print sequence,"not recognized" 01107 raise 01108 return l
| def ConfigBuilder::ConfigBuilder::prepare | ( | self, | |
doChecking = False |
|||
| ) |
Prepare the configuration string and add missing pieces.
Definition at line 1854 of file ConfigBuilder.py.
01855 : 01856 """ Prepare the configuration string and add missing pieces.""" 01857 01858 self.loadAndRemember(self.EVTCONTDefaultCFF) #load the event contents regardless 01859 self.addMaxEvents() 01860 if self.with_input: 01861 self.addSource() 01862 self.addStandardSequences() 01863 ##adding standard sequences might change the inputEventContent option and therefore needs to be finalized after 01864 self.completeInputCommand() 01865 01866 self.addConditions() 01867 01868 01869 outputModuleCfgCode="" 01870 if not 'HARVESTING' in self.stepMap.keys() and not 'SKIM' in self.stepMap.keys() and not 'ALCAHARVEST' in self.stepMap.keys() and not 'ALCAOUTPUT' in self.stepMap.keys() and self.with_output: 01871 outputModuleCfgCode=self.addOutput() 01872 01873 self.addCommon() 01874 01875 self.pythonCfgCode = "# Auto generated configuration file\n" 01876 self.pythonCfgCode += "# using: \n# "+__version__[1:-1]+"\n# "+__source__[1:-1]+'\n' 01877 self.pythonCfgCode += "# with command line options: "+self._options.arguments+'\n' 01878 self.pythonCfgCode += "import FWCore.ParameterSet.Config as cms\n\n" 01879 self.pythonCfgCode += "process = cms.Process('"+self.process.name_()+"')\n\n" 01880 01881 self.pythonCfgCode += "# import of standard configurations\n" 01882 for module in self.imports: 01883 self.pythonCfgCode += ("process.load('"+module+"')\n") 01884 01885 # production info 01886 if not hasattr(self.process,"configurationMetadata"): 01887 self.build_production_info(self._options.evt_type, self._options.number) 01888 else: 01889 #the PSet was added via a load 01890 self.addedObjects.append(("Production Info","configurationMetadata")) 01891 01892 self.pythonCfgCode +="\n" 01893 for comment,object in self.addedObjects: 01894 if comment!="": 01895 self.pythonCfgCode += "\n# "+comment+"\n" 01896 self.pythonCfgCode += dumpPython(self.process,object) 01897 01898 # dump the output definition 01899 self.pythonCfgCode += "\n# Output definition\n" 01900 self.pythonCfgCode += outputModuleCfgCode 01901 01902 # dump all additional outputs (e.g. alca or skim streams) 01903 self.pythonCfgCode += "\n# Additional output definition\n" 01904 #I do not understand why the keys are not normally ordered. 01905 nl=self.additionalOutputs.keys() 01906 nl.sort() 01907 for name in nl: 01908 output = self.additionalOutputs[name] 01909 self.pythonCfgCode += "process.%s = %s" %(name, output.dumpPython()) 01910 tmpOut = cms.EndPath(output) 01911 setattr(self.process,name+'OutPath',tmpOut) 01912 self.schedule.append(tmpOut) 01913 01914 # dump all additional commands 01915 self.pythonCfgCode += "\n# Other statements\n" 01916 for command in self.additionalCommands: 01917 self.pythonCfgCode += command + "\n" 01918 01919 #comma separated list of objects that deserve to be inlined in the configuration (typically from a modified config deep down) 01920 for object in self._options.inlineObjets.split(','): 01921 if not object: 01922 continue 01923 if not hasattr(self.process,object): 01924 print 'cannot inline -'+object+'- : not known' 01925 else: 01926 self.pythonCfgCode +='\n' 01927 self.pythonCfgCode +=dumpPython(self.process,object) 01928 01929 # dump all paths 01930 self.pythonCfgCode += "\n# Path and EndPath definitions\n" 01931 for path in self.process.paths: 01932 if getattr(self.process,path) not in self.blacklist_paths: 01933 self.pythonCfgCode += dumpPython(self.process,path) 01934 01935 for endpath in self.process.endpaths: 01936 if getattr(self.process,endpath) not in self.blacklist_paths: 01937 self.pythonCfgCode += dumpPython(self.process,endpath) 01938 01939 # dump the schedule 01940 self.pythonCfgCode += "\n# Schedule definition\n" 01941 result = "process.schedule = cms.Schedule(" 01942 01943 # handling of the schedule 01944 self.process.schedule = cms.Schedule() 01945 for item in self.schedule: 01946 if not isinstance(item, cms.Schedule): 01947 self.process.schedule.append(item) 01948 else: 01949 self.process.schedule.extend(item) 01950 01951 if hasattr(self.process,"HLTSchedule"): 01952 beforeHLT = self.schedule[:self.schedule.index(self.process.HLTSchedule)] 01953 afterHLT = self.schedule[self.schedule.index(self.process.HLTSchedule)+1:] 01954 pathNames = ['process.'+p.label_() for p in beforeHLT] 01955 result += ','.join(pathNames)+')\n' 01956 result += 'process.schedule.extend(process.HLTSchedule)\n' 01957 pathNames = ['process.'+p.label_() for p in afterHLT] 01958 result += 'process.schedule.extend(['+','.join(pathNames)+'])\n' 01959 else: 01960 pathNames = ['process.'+p.label_() for p in self.schedule] 01961 result ='process.schedule = cms.Schedule('+','.join(pathNames)+')\n' 01962 01963 self.pythonCfgCode += result 01964 01965 #repacked version 01966 if self._options.isRepacked: 01967 self.pythonCfgCode +="\n" 01968 self.pythonCfgCode +="from Configuration.PyReleaseValidation.ConfigBuilder import MassReplaceInputTag\n" 01969 self.pythonCfgCode +="MassReplaceInputTag(process)\n" 01970 MassReplaceInputTag(self.process) 01971 01972 # special treatment in case of production filter sequence 2/2 01973 if self.productionFilterSequence: 01974 self.pythonCfgCode +='# filter all path with the production filter sequence\n' 01975 self.pythonCfgCode +='for path in process.paths:\n' 01976 if len(self.conditionalPaths): 01977 self.pythonCfgCode +='\tif not path in %s: continue\n'%str(self.conditionalPaths) 01978 self.pythonCfgCode +='\tgetattr(process,path)._seq = process.%s * getattr(process,path)._seq \n'%(self.productionFilterSequence,) 01979 pfs = getattr(self.process,self.productionFilterSequence) 01980 for path in self.process.paths: 01981 if not path in self.conditionalPaths: continue 01982 getattr(self.process,path)._seq = pfs * getattr(self.process,path)._seq 01983 01984 # dump customise fragment 01985 self.pythonCfgCode += self.addCustomise() 01986 01987 # make the .io file 01988 01989 if self._options.io: 01990 #io=open(self._options.python_filename.replace('.py','.io'),'w') 01991 if not self._options.io.endswith('.io'): self._option.io+='.io' 01992 io=open(self._options.io,'w') 01993 ioJson={} 01994 if hasattr(self.process.source,"fileNames"): 01995 if len(self.process.source.fileNames.value()): 01996 ioJson['primary']=self.process.source.fileNames.value() 01997 if hasattr(self.process.source,"secondaryFileNames"): 01998 if len(self.process.source.secondaryFileNames.value()): 01999 ioJson['secondary']=self.process.source.secondaryFileNames.value() 02000 02001 for (o,om) in self.process.outputModules_().items(): 02002 ioJson[o]=om.fileName.value() 02003 #ioJson['GT']=self._options.conditions 02004 ioJson['GT']=self.process.GlobalTag.globaltag.value() 02005 if self.productionFilterSequence: 02006 ioJson['filter']=self.productionFilterSequence 02007 import json 02008 io.write(json.dumps(ioJson)) 02009 return 02010 02011 02012
| def ConfigBuilder::ConfigBuilder::prepare_ALCA | ( | self, | |
sequence = None, |
|||
workflow = 'full' |
|||
| ) |
Enrich the process with alca streams
Definition at line 1146 of file ConfigBuilder.py.
01147 : 01148 """ Enrich the process with alca streams """ 01149 alcaConfig=self.loadDefaultOrSpecifiedCFF(sequence,self.ALCADefaultCFF) 01150 sequence = sequence.split('.')[-1] 01151 01152 # decide which ALCA paths to use 01153 alcaList = sequence.split("+") 01154 maxLevel=0 01155 from Configuration.AlCa.autoAlca import autoAlca 01156 # support @X from autoAlca.py, and recursion support: i.e T0:@Mu+@EG+... 01157 self.expandMapping(alcaList,autoAlca) 01158 01159 for name in alcaConfig.__dict__: 01160 alcastream = getattr(alcaConfig,name) 01161 shortName = name.replace('ALCARECOStream','') 01162 if shortName in alcaList and isinstance(alcastream,cms.FilteredStream): 01163 output = self.addExtraStream(name,alcastream, workflow = workflow) 01164 if 'DQM' in alcaList: 01165 if not self._options.inlineEventContent and hasattr(self.process,name): 01166 self.executeAndRemember('process.' + name + '.outputCommands.append("keep *_MEtoEDMConverter_*_*")') 01167 else: 01168 output.outputCommands.append("keep *_MEtoEDMConverter_*_*") 01169 01170 #rename the HLT process name in the alca modules 01171 if self._options.hltProcess or 'HLT' in self.stepMap: 01172 if isinstance(alcastream.paths,tuple): 01173 for path in alcastream.paths: 01174 self.renameHLTprocessInSequence(path.label()) 01175 else: 01176 self.renameHLTprocessInSequence(alcastream.paths.label()) 01177 01178 for i in range(alcaList.count(shortName)): 01179 alcaList.remove(shortName) 01180 01181 # DQM needs a special handling 01182 elif name == 'pathALCARECODQM' and 'DQM' in alcaList: 01183 path = getattr(alcaConfig,name) 01184 self.schedule.append(path) 01185 alcaList.remove('DQM') 01186 01187 if isinstance(alcastream,cms.Path): 01188 #black list the alca path so that they do not appear in the cfg 01189 self.blacklist_paths.append(alcastream) 01190 01191 01192 if len(alcaList) != 0: 01193 available=[] 01194 for name in alcaConfig.__dict__: 01195 alcastream = getattr(alcaConfig,name) 01196 if isinstance(alcastream,cms.FilteredStream): 01197 available.append(name.replace('ALCARECOStream','')) 01198 print "The following alcas could not be found "+str(alcaList) 01199 print "available ",available 01200 #print "verify your configuration, ignoring for now" 01201 raise Exception("The following alcas could not be found "+str(alcaList))
| def ConfigBuilder::ConfigBuilder::prepare_ALCAHARVEST | ( | self, | |
sequence = None |
|||
| ) |
Enrich the process with AlCaHarvesting step
Definition at line 1765 of file ConfigBuilder.py.
01766 : 01767 """ Enrich the process with AlCaHarvesting step """ 01768 harvestingConfig = self.loadAndRemember(self.ALCAHARVESTDefaultCFF) 01769 sequence=sequence.split(".")[-1] 01770 01771 # decide which AlcaHARVESTING paths to use 01772 harvestingList = sequence.split("+") 01773 for name in harvestingConfig.__dict__: 01774 harvestingstream = getattr(harvestingConfig,name) 01775 if name in harvestingList and isinstance(harvestingstream,cms.Path): 01776 self.schedule.append(harvestingstream) 01777 self.executeAndRemember("process.PoolDBOutputService.toPut.append(process.ALCAHARVEST" + name + "_dbOutput)") 01778 self.executeAndRemember("process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVEST" + name + "_metadata)") 01779 harvestingList.remove(name) 01780 # append the common part at the end of the sequence 01781 lastStep = getattr(harvestingConfig,"ALCAHARVESTDQMSaveAndMetadataWriter") 01782 self.schedule.append(lastStep) 01783 01784 if len(harvestingList) != 0 and 'dummyHarvesting' not in harvestingList : 01785 print "The following harvesting could not be found : ", harvestingList 01786 raise Exception("The following harvesting could not be found : "+str(harvestingList)) 01787 01788
| def ConfigBuilder::ConfigBuilder::prepare_ALCAOUTPUT | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1143 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_ALCAPRODUCER | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1140 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_CFWRITER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the crossing frame writer step
Definition at line 1325 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DATAMIX | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the digitisation step
Definition at line 1331 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DIGI | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the digitisation step
Definition at line 1309 of file ConfigBuilder.py.
01310 : 01311 """ Enrich the schedule with the digitisation step""" 01312 self.loadDefaultOrSpecifiedCFF(sequence,self.DIGIDefaultCFF) 01313 01314 if self._options.gflash==True: 01315 self.loadAndRemember("Configuration/StandardSequences/GFlashDIGI_cff") 01316 01317 if self._options.himix==True: 01318 self.loadAndRemember("SimGeneral/MixingModule/himixDIGI_cff") 01319 01320 if self._options.restoreRNDSeeds: 01321 self.executeAndRemember("process.mix.playback = True") 01322 01323 self.scheduleSequence(sequence.split('.')[-1],'digitisation_step') 01324 return
| def ConfigBuilder::ConfigBuilder::prepare_DIGI2RAW | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1337 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DQM | ( | self, | |
sequence = 'DQMOffline' |
|||
| ) |
Definition at line 1699 of file ConfigBuilder.py.
01700 : 01701 # this one needs replacement 01702 01703 self.loadDefaultOrSpecifiedCFF(sequence,self.DQMOFFLINEDefaultCFF) 01704 sequenceList=sequence.split('.')[-1].split('+') 01705 from DQMOffline.Configuration.autoDQM import autoDQM 01706 self.expandMapping(sequenceList,autoDQM,index=0) 01707 01708 if len(set(sequenceList))!=len(sequenceList): 01709 sequenceList=list(set(sequenceList)) 01710 print "Duplicate entries for DQM:, using",sequenceList 01711 pathName='dqmoffline_step' 01712 01713 for (i,sequence) in enumerate(sequenceList): 01714 if (i!=0): 01715 pathName='dqmoffline_%d_step'%(i) 01716 01717 if 'HLT' in self.stepMap.keys() or self._options.hltProcess: 01718 self.renameHLTprocessInSequence(sequence) 01719 01720 # if both HLT and DQM are run in the same process, schedule [HLT]DQM in an EndPath 01721 if 'HLT' in self.stepMap.keys(): 01722 # need to put [HLT]DQM in an EndPath, to access the HLT trigger results 01723 #setattr(self.process,pathName, cms.EndPath( getattr(self.process, sequence ) ) ) 01724 self.scheduleSequence(sequence, pathName, what='EndPath') 01725 01726 else: 01727 # schedule DQM as a standard Path 01728 #setattr(self.process,pathName, cms.Path( getattr(self.process, sequence) ) ) 01729 self.scheduleSequence(sequence, pathName, what='Path') 01730 #self.schedule.append(getattr(self.process,pathName)) 01731
| def ConfigBuilder::ConfigBuilder::prepare_ENDJOB | ( | self, | |
sequence = 'endOfProcess' |
|||
| ) |
Definition at line 1789 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_FASTSIM | ( | self, | |
sequence = "all" |
|||
| ) |
Enrich the schedule with fastsim
Definition at line 1798 of file ConfigBuilder.py.
01799 : 01800 """Enrich the schedule with fastsim""" 01801 self.loadAndRemember("FastSimulation/Configuration/FamosSequences_cff") 01802 01803 if sequence in ('all','allWithHLTFiltering',''): 01804 if not 'HLT' in self.stepMap.keys(): 01805 self.prepare_HLT(sequence=None) 01806 01807 self.executeAndRemember("process.famosSimHits.SimulateCalorimetry = True") 01808 self.executeAndRemember("process.famosSimHits.SimulateTracking = True") 01809 01810 self.executeAndRemember("process.simulation = cms.Sequence(process.simulationWithFamos)") 01811 self.executeAndRemember("process.HLTEndSequence = cms.Sequence(process.reconstructionWithFamos)") 01812 01813 # since we have HLT here, the process should be called HLT 01814 self._options.name = "HLT" 01815 01816 # if we don't want to filter after HLT but simulate everything regardless of what HLT tells, we have to add reconstruction explicitly 01817 if sequence == 'all' and not 'HLT' in self.stepMap.keys(): #(a) 01818 self.finalizeFastSimHLT() 01819 elif sequence == 'famosWithEverything': 01820 self.process.fastsim_step = cms.Path( getattr(self.process, "famosWithEverything") ) 01821 self.schedule.append(self.process.fastsim_step) 01822 01823 # now the additional commands we need to make the config work 01824 self.executeAndRemember("process.VolumeBasedMagneticFieldESProducer.useParametrizedTrackerField = True") 01825 else: 01826 print "FastSim setting", sequence, "unknown." 01827 raise ValueError 01828 01829 if 'Flat' in self._options.beamspot: 01830 beamspotType = 'Flat' 01831 elif 'Gauss' in self._options.beamspot: 01832 beamspotType = 'Gaussian' 01833 else: 01834 beamspotType = 'BetaFunc' 01835 self.loadAndRemember('IOMC.EventVertexGenerators.VtxSmearedParameters_cfi') 01836 beamspotName = 'process.%sVtxSmearingParameters' %(self._options.beamspot) 01837 self.executeAndRemember(beamspotName+'.type = cms.string("%s")'%(beamspotType)) 01838 self.executeAndRemember('process.famosSimHits.VertexGenerator = '+beamspotName) 01839 if hasattr(self.process,'famosPileUp'): 01840 self.executeAndRemember('process.famosPileUp.VertexGenerator = '+beamspotName) 01841 01842
| def ConfigBuilder::ConfigBuilder::prepare_FILTER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with a user defined filter sequence
Definition at line 1458 of file ConfigBuilder.py.
01459 : 01460 ''' Enrich the schedule with a user defined filter sequence ''' 01461 ## load the relevant part 01462 filterConfig=self.load(sequence.split('.')[0]) 01463 filterSeq=sequence.split('.')[-1] 01464 ## print it in the configuration 01465 class PrintAllModules(object): 01466 def __init__(self): 01467 self.inliner='' 01468 pass 01469 def enter(self,visitee): 01470 try: 01471 label=visitee.label() 01472 ##needs to be in reverse order 01473 self.inliner=label+','+self.inliner 01474 except: 01475 pass 01476 def leave(self,v): pass 01477 01478 expander=PrintAllModules() 01479 getattr(self.process,filterSeq).visit( expander ) 01480 self._options.inlineObjets+=','+expander.inliner 01481 self._options.inlineObjets+=','+filterSeq 01482 01483 ## put the filtering path in the schedule 01484 self.scheduleSequence(filterSeq,'filtering_step') 01485 self.nextScheduleIsConditional=True 01486 ## put it before all the other paths 01487 self.productionFilterSequence = filterSeq 01488 01489 return
| def ConfigBuilder::ConfigBuilder::prepare_GEN | ( | self, | |
sequence = None |
|||
| ) |
load the fragment of generator configuration
Definition at line 1218 of file ConfigBuilder.py.
01219 : 01220 """ load the fragment of generator configuration """ 01221 loadFailure=False 01222 #remove trailing .py 01223 #support old style .cfi by changing into something.cfi into something_cfi 01224 #remove python/ from the name 01225 loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','') 01226 #standard location of fragments 01227 if not '/' in loadFragment: 01228 loadFragment='Configuration.Generator.'+loadFragment 01229 else: 01230 loadFragment=loadFragment.replace('/','.') 01231 try: 01232 print "Loading generator fragment from",loadFragment 01233 __import__(loadFragment) 01234 except: 01235 loadFailure=True 01236 #if self.process.source and self.process.source.type_()=='EmptySource': 01237 if not (self._options.filein or self._options.dbsquery): 01238 raise Exception("Neither gen fragment of input files provided: this is an inconsistent GEN step configuration") 01239 01240 if not loadFailure: 01241 generatorModule=sys.modules[loadFragment] 01242 genModules=generatorModule.__dict__ 01243 if self._options.hideGen: 01244 self.loadAndRemember(loadFragment) 01245 else: 01246 self.process.load(loadFragment) 01247 # expose the objects from that fragment to the configuration 01248 import FWCore.ParameterSet.Modules as cmstypes 01249 for name in genModules: 01250 theObject = getattr(generatorModule,name) 01251 if isinstance(theObject, cmstypes._Module): 01252 self._options.inlineObjets=name+','+self._options.inlineObjets 01253 elif isinstance(theObject, cms.Sequence) or isinstance(theObject, cmstypes.ESProducer): 01254 self._options.inlineObjets+=','+name 01255 01256 if sequence == self.GENDefaultSeq or sequence == 'pgen_genonly' or ( sequence == 'pgen_hi' and 'generator' in genModules): 01257 if 'ProductionFilterSequence' in genModules and ('generator' in genModules or 'hiSignal' in genModules): 01258 self.productionFilterSequence = 'ProductionFilterSequence' 01259 elif 'generator' in genModules: 01260 self.productionFilterSequence = 'generator' 01261 01262 """ Enrich the schedule with the rest of the generation step """ 01263 self.loadDefaultOrSpecifiedCFF(sequence,self.GENDefaultCFF) 01264 genSeqName=sequence.split('.')[-1] 01265 01266 if not 'FASTSIM' in self.stepMap: 01267 try: 01268 from Configuration.StandardSequences.VtxSmeared import VtxSmeared 01269 self.loadAndRemember(VtxSmeared[self._options.beamspot]) 01270 except ImportError: 01271 raise Exception("VertexSmearing type or beamspot "+self._options.beamspot+" unknown.") 01272 01273 if self._options.scenario == 'HeavyIons' and self._options.himix: 01274 self.loadAndRemember("SimGeneral/MixingModule/himixGEN_cff") 01275 01276 self.process.generation_step = cms.Path( getattr(self.process,genSeqName) ) 01277 self.schedule.append(self.process.generation_step) 01278 01279 #register to the genstepfilter the name of the path (static right now, but might evolve) 01280 self.executeAndRemember('process.genstepfilter.triggerConditions=cms.vstring("generation_step")') 01281 01282 if 'reGEN' in self.stepMap: 01283 #stop here 01284 return 01285 01286 """ Enrich the schedule with the summary of the filter step """ 01287 #the gen filter in the endpath 01288 self.loadAndRemember("GeneratorInterface/Core/genFilterSummary_cff") 01289 self.scheduleSequenceAtEnd('genFilterSummary','genfiltersummary_step') 01290 return
| def ConfigBuilder::ConfigBuilder::prepare_HARVESTING | ( | self, | |
sequence = None |
|||
| ) |
Enrich the process with harvesting step
Definition at line 1732 of file ConfigBuilder.py.
01733 : 01734 """ Enrich the process with harvesting step """ 01735 self.EDMtoMECFF='Configuration/StandardSequences/EDMtoME'+self._options.harvesting+'_cff' 01736 self.loadAndRemember(self.EDMtoMECFF) 01737 self.scheduleSequence('EDMtoME','edmtome_step') 01738 01739 harvestingConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.HARVESTINGDefaultCFF) 01740 sequence = sequence.split('.')[-1] 01741 01742 # decide which HARVESTING paths to use 01743 harvestingList = sequence.split("+") 01744 from DQMOffline.Configuration.autoDQM import autoDQM 01745 self.expandMapping(harvestingList,autoDQM,index=1) 01746 01747 if len(set(harvestingList))!=len(harvestingList): 01748 harvestingList=list(set(harvestingList)) 01749 print "Duplicate entries for HARVESTING, using",harvestingList 01750 01751 for name in harvestingList: 01752 if not name in harvestingConfig.__dict__: 01753 print name,"is not a possible harvesting type. Available are",harvestingConfig.__dict__.keys() 01754 continue 01755 harvestingstream = getattr(harvestingConfig,name) 01756 if isinstance(harvestingstream,cms.Path): 01757 self.schedule.append(harvestingstream) 01758 self.blacklist_paths.append(harvestingstream) 01759 if isinstance(harvestingstream,cms.Sequence): 01760 setattr(self.process,name+"_step",cms.Path(harvestingstream)) 01761 self.schedule.append(getattr(self.process,name+"_step")) 01762 01763 self.scheduleSequence('DQMSaver','dqmsave_step') 01764 return
| def ConfigBuilder::ConfigBuilder::prepare_HLT | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the HLT simulation step
Definition at line 1372 of file ConfigBuilder.py.
01373 : 01374 """ Enrich the schedule with the HLT simulation step""" 01375 if not sequence: 01376 print "no specification of the hlt menu has been given, should never happen" 01377 raise Exception('no HLT sequence provided') 01378 01379 if '@' in sequence: 01380 # case where HLT:@something was provided 01381 from Configuration.HLT.autoHLT import autoHLT 01382 key = sequence[1:] 01383 if key in autoHLT: 01384 sequence = autoHLT[key] 01385 else: 01386 raise ValueError('no HLT mapping key "%s" found in autoHLT' % key) 01387 01388 if ',' in sequence: 01389 # case where HLT:something:something was provided transcribed in sequence = something,something 01390 self.executeAndRemember('import HLTrigger.Configuration.Utilities') 01391 optionsForHLT = {} 01392 if self._options.scenario == 'HeavyIons': 01393 optionsForHLT['type'] = 'HIon' 01394 else: 01395 optionsForHLT['type'] = 'GRun' 01396 optionsForHLTConfig = ', '.join('%s=%s' % (key, repr(val)) for (key, val) in optionsForHLT.iteritems()) 01397 if sequence == 'run,fromSource': 01398 if hasattr(self.process.source,'firstRun'): 01399 self.executeAndRemember('process.loadHltConfiguration("run:%%d"%%(process.source.firstRun.value()),%s)'%(optionsForHLTConfig)) 01400 elif hasattr(self.process.source,'setRunNumber'): 01401 self.executeAndRemember('process.loadHltConfiguration("run:%%d"%%(process.source.setRunNumber.value()),%s)'%(optionsForHLTConfig)) 01402 else: 01403 raise Exception('Cannot replace menu to load %s'%(sequence)) 01404 else: 01405 self.executeAndRemember('process.loadHltConfiguration("%s",%s)'%(sequence.replace(',',':'),optionsForHLTConfig)) 01406 else: 01407 if 'FASTSIM' in self.stepMap: 01408 self.loadAndRemember('HLTrigger/Configuration/HLT_%s_Famos_cff' % sequence) 01409 else: 01410 self.loadAndRemember('HLTrigger/Configuration/HLT_%s_cff' % sequence) 01411 01412 if self._options.isMC: 01413 self._options.customisation_file+=",HLTrigger/Configuration/customizeHLTforMC.customizeHLTforMC" 01414 01415 if self._options.name != 'HLT': 01416 self.additionalCommands.append('from HLTrigger.Configuration.CustomConfigs import ProcessName') 01417 self.additionalCommands.append('process = ProcessName(process)') 01418 self.additionalCommands.append('') 01419 from HLTrigger.Configuration.CustomConfigs import ProcessName 01420 self.process = ProcessName(self.process) 01421 01422 self.schedule.append(self.process.HLTSchedule) 01423 [self.blacklist_paths.append(path) for path in self.process.HLTSchedule if isinstance(path,(cms.Path,cms.EndPath))] 01424 if ('FASTSIM' in self.stepMap and 'HLT' in self.stepMap): 01425 self.finalizeFastSimHLT() 01426
| def ConfigBuilder::ConfigBuilder::prepare_L1 | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the L1 simulation step
Definition at line 1347 of file ConfigBuilder.py.
01348 : 01349 """ Enrich the schedule with the L1 simulation step""" 01350 if not sequence: 01351 self.loadAndRemember(self.L1EMDefaultCFF) 01352 else: 01353 # let the L1 package decide for the scenarios available 01354 from L1Trigger.Configuration.ConfigBuilder import getConfigsForScenario 01355 listOfImports = getConfigsForScenario(sequence) 01356 for file in listOfImports: 01357 self.loadAndRemember(file) 01358 self.scheduleSequence('SimL1Emulator','L1simulation_step') 01359 return
| def ConfigBuilder::ConfigBuilder::prepare_L1HwVal | ( | self, | |
sequence = 'L1HwVal' |
|||
| ) |
Enrich the schedule with L1 HW validation
Definition at line 1445 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_L1Reco | ( | self, | |
sequence = "L1Reco" |
|||
| ) |
Enrich the schedule with L1 reconstruction
Definition at line 1452 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_L1REPACK | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection
Definition at line 1360 of file ConfigBuilder.py.
01361 : 01362 """ Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection""" 01363 if sequence is not 'GT': 01364 print 'Running the full L1 emulator is not supported yet' 01365 raise Exception('unsupported feature') 01366 if sequence is 'GT': 01367 self.loadAndRemember('Configuration/StandardSequences/SimL1EmulatorRepack_GT_cff') 01368 if self._options.scenario == 'HeavyIons': 01369 self.renameInputTagsInSequence("SimL1Emulator","rawDataCollector","rawDataRepacker") 01370 self.scheduleSequence('SimL1Emulator','L1simulation_step') 01371
| def ConfigBuilder::ConfigBuilder::prepare_LHE | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1202 of file ConfigBuilder.py.
01203 : 01204 #load the fragment 01205 ##make it loadable 01206 loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','').replace('/','.') 01207 print "Loading lhe fragment from",loadFragment 01208 __import__(loadFragment) 01209 self.process.load(loadFragment) 01210 ##inline the modules 01211 self._options.inlineObjets+=','+sequence 01212 01213 getattr(self.process,sequence).nEvents = int(self._options.number) 01214 01215 #schedule it 01216 self.process.lhe_step = cms.Path( getattr( self.process,sequence) ) 01217 self.schedule.append( self.process.lhe_step )
| def ConfigBuilder::ConfigBuilder::prepare_POSTRECO | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the postreco step
Definition at line 1546 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RAW2DIGI | ( | self, | |
sequence = "RawToDigi" |
|||
| ) |
Definition at line 1438 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RAW2RECO | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1427 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RECO | ( | self, | |
sequence = "reconstruction" |
|||
| ) |
Enrich the schedule with reconstruction
Definition at line 1490 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_REPACK | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1342 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_SIM | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the simulation step
Definition at line 1291 of file ConfigBuilder.py.
01292 : 01293 """ Enrich the schedule with the simulation step""" 01294 self.loadAndRemember(self.SIMDefaultCFF) 01295 if self._options.gflash==True: 01296 self.loadAndRemember("Configuration/StandardSequences/GFlashSIM_cff") 01297 01298 if self._options.magField=='0T': 01299 self.executeAndRemember("process.g4SimHits.UseMagneticField = cms.bool(False)") 01300 01301 if self._options.himix==True: 01302 if self._options.geometry in defaultOptions.geometryExtendedOptions: 01303 self.loadAndRemember("SimGeneral/MixingModule/himixSIMExtended_cff") 01304 else: 01305 self.loadAndRemember("SimGeneral/MixingModule/himixSIMIdeal_cff") 01306 01307 self.scheduleSequence('psim','simulation_step') 01308 return
| def ConfigBuilder::ConfigBuilder::prepare_SKIM | ( | self, | |
sequence = "all" |
|||
| ) |
Enrich the schedule with skimming fragments
Definition at line 1496 of file ConfigBuilder.py.
01497 : 01498 ''' Enrich the schedule with skimming fragments''' 01499 skimConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.SKIMDefaultCFF) 01500 sequence = sequence.split('.')[-1] 01501 01502 skimlist=sequence.split('+') 01503 ## support @Mu+DiJet+@Electron configuration via autoSkim.py 01504 from Configuration.Skimming.autoSkim import autoSkim 01505 self.expandMapping(skimlist,autoSkim) 01506 01507 #print "dictionnary for skims:",skimConfig.__dict__ 01508 for skim in skimConfig.__dict__: 01509 skimstream = getattr(skimConfig,skim) 01510 if isinstance(skimstream,cms.Path): 01511 #black list the alca path so that they do not appear in the cfg 01512 self.blacklist_paths.append(skimstream) 01513 if (not isinstance(skimstream,cms.FilteredStream)): 01514 continue 01515 shortname = skim.replace('SKIMStream','') 01516 if (sequence=="all"): 01517 self.addExtraStream(skim,skimstream) 01518 elif (shortname in skimlist): 01519 self.addExtraStream(skim,skimstream) 01520 #add a DQM eventcontent for this guy 01521 if self._options.datatier!="": 01522 self.process.load(self.EVTCONTDefaultCFF) 01523 skimstreamDQM = cms.FilteredStream( 01524 responsible = skimstream.responsible, 01525 name = skimstream.name+'DQM', 01526 paths = skimstream.paths, 01527 selectEvents = skimstream.selectEvents, 01528 content = self._options.datatier+'EventContent', 01529 dataTier = cms.untracked.string(self._options.datatier) 01530 ) 01531 self.addExtraStream(skim+'DQM',skimstreamDQM) 01532 for i in range(skimlist.count(shortname)): 01533 skimlist.remove(shortname) 01534 01535 01536 01537 if (skimlist.__len__()!=0 and sequence!="all"): 01538 print 'WARNING, possible typo with SKIM:'+'+'.join(skimlist) 01539 raise Exception('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
| def ConfigBuilder::ConfigBuilder::prepare_USER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with a user defined sequence
Definition at line 1540 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_VALIDATION | ( | self, | |
sequence = 'validation' |
|||
| ) |
Definition at line 1553 of file ConfigBuilder.py.
01554 : 01555 self.loadDefaultOrSpecifiedCFF(sequence,self.VALIDATIONDefaultCFF) 01556 #in case VALIDATION:something:somethingelse -> something,somethingelse 01557 sequence=sequence.split('.')[-1] 01558 if sequence.find(',')!=-1: 01559 prevalSeqName=sequence.split(',')[0] 01560 valSeqName=sequence.split(',')[1] 01561 else: 01562 postfix='' 01563 if sequence: 01564 postfix='_'+sequence 01565 prevalSeqName='prevalidation'+postfix 01566 valSeqName='validation'+postfix 01567 if not hasattr(self.process,valSeqName): 01568 prevalSeqName='' 01569 valSeqName=sequence 01570 01571 if not 'DIGI' in self.stepMap and not 'FASTSIM' in self.stepMap and not valSeqName.startswith('genvalid'): 01572 if self._options.restoreRNDSeeds==False and not self._options.restoreRNDSeeds==True: 01573 self._options.restoreRNDSeeds=True 01574 01575 #rename the HLT process in validation steps 01576 if ('HLT' in self.stepMap and not 'FASTSIM' in self.stepMap) or self._options.hltProcess: 01577 self.renameHLTprocessInSequence(valSeqName) 01578 if prevalSeqName: 01579 self.renameHLTprocessInSequence(prevalSeqName) 01580 01581 if prevalSeqName: 01582 self.process.prevalidation_step = cms.Path( getattr(self.process, prevalSeqName ) ) 01583 self.schedule.append(self.process.prevalidation_step) 01584 01585 self.process.validation_step = cms.EndPath( getattr(self.process,valSeqName ) ) 01586 self.schedule.append(self.process.validation_step) 01587 01588 if not 'DIGI' in self.stepMap and not 'FASTSIM' in self.stepMap: 01589 self.executeAndRemember("process.mix.playback = True") 01590 01591 if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions): 01592 #will get in the schedule, smoothly 01593 self.process.validation_step._seq = self.process.genstepfilter * self.process.validation_step._seq 01594 01595 return 01596
| def ConfigBuilder::ConfigBuilder::profileOptions | ( | self | ) |
addIgProfService Function to add the igprof profile service so that you can dump in the middle of the run.
Definition at line 219 of file ConfigBuilder.py.
00220 : 00221 """ 00222 addIgProfService 00223 Function to add the igprof profile service so that you can dump in the middle 00224 of the run. 00225 """ 00226 profileOpts = self._options.profile.split(':') 00227 profilerStart = 1 00228 profilerInterval = 100 00229 profilerFormat = None 00230 profilerJobFormat = None 00231 00232 if len(profileOpts): 00233 #type, given as first argument is unused here 00234 profileOpts.pop(0) 00235 if len(profileOpts): 00236 startEvent = profileOpts.pop(0) 00237 if not startEvent.isdigit(): 00238 raise Exception("%s is not a number" % startEvent) 00239 profilerStart = int(startEvent) 00240 if len(profileOpts): 00241 eventInterval = profileOpts.pop(0) 00242 if not eventInterval.isdigit(): 00243 raise Exception("%s is not a number" % eventInterval) 00244 profilerInterval = int(eventInterval) 00245 if len(profileOpts): 00246 profilerFormat = profileOpts.pop(0) 00247 00248 00249 if not profilerFormat: 00250 profilerFormat = "%s___%s___%s___%s___%s___%s___%%I.gz" % (self._options.evt_type.replace("_cfi", ""), 00251 self._options.step, 00252 self._options.pileup, 00253 self._options.conditions, 00254 self._options.datatier, 00255 self._options.profileTypeLabel) 00256 if not profilerJobFormat and profilerFormat.endswith(".gz"): 00257 profilerJobFormat = profilerFormat.replace(".gz", "_EndOfJob.gz") 00258 elif not profilerJobFormat: 00259 profilerJobFormat = profilerFormat + "_EndOfJob.gz" 00260 00261 return (profilerStart,profilerInterval,profilerFormat,profilerJobFormat)
| def ConfigBuilder::ConfigBuilder::renameHLTprocessInSequence | ( | self, | |
| sequence, | |||
proc = None, |
|||
HLTprocess = 'HLT' |
|||
| ) |
Definition at line 1666 of file ConfigBuilder.py.
01667 : 01668 if self._options.hltProcess: 01669 proc=self._options.hltProcess 01670 else: 01671 proc=self.process.name_() 01672 if proc==HLTprocess: return 01673 # look up all module in dqm sequence 01674 print "replacing %s process name - sequence %s will use '%s'" % (HLTprocess,sequence, proc) 01675 getattr(self.process,sequence).visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor(HLTprocess,proc,whitelist = ("subSystemFolder",))) 01676 if 'from Configuration.PyReleaseValidation.ConfigBuilder import ConfigBuilder' not in self.additionalCommands: 01677 self.additionalCommands.append('from Configuration.PyReleaseValidation.ConfigBuilder import ConfigBuilder') 01678 self.additionalCommands.append('process.%s.visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor("%s", "%s", whitelist = ("subSystemFolder",)))'% (sequence,HLTprocess, proc)) 01679
| def ConfigBuilder::ConfigBuilder::renameInputTagsInSequence | ( | self, | |
| sequence, | |||
oldT = "rawDataCollector", |
|||
newT = "rawDataRepacker" |
|||
| ) |
Definition at line 1656 of file ConfigBuilder.py.
01657 : 01658 print "Replacing all InputTag %s => %s"%(oldT,newT) 01659 from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag 01660 massSearchReplaceAnyInputTag(getattr(self.process,sequence),oldT,newT) 01661 loadMe='from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag' 01662 if not loadMe in self.additionalCommands: 01663 self.additionalCommands.append(loadMe) 01664 self.additionalCommands.append('massSearchReplaceAnyInputTag(process.%s,"%s","%s",False)'%(sequence,oldT,newT))
| def ConfigBuilder::ConfigBuilder::scheduleSequence | ( | self, | |
| seq, | |||
| prefix, | |||
what = 'Path' |
|||
| ) |
Definition at line 1109 of file ConfigBuilder.py.
01110 : 01111 if not what in ['Path','EndPath']: 01112 raise Exception("Can only schedule a Path or and EndPath."+str(what)+"is not allowed") 01113 if '*' in seq: 01114 #create only one path with all sequences in it 01115 for i,s in enumerate(seq.split('*')): 01116 if i==0: 01117 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, s) )) 01118 else: 01119 p=getattr(self.process,prefix) 01120 p+=getattr(self.process, s) 01121 self.schedule.append(getattr(self.process,prefix)) 01122 return 01123 else: 01124 #create as many path as many sequences 01125 if not '+' in seq: 01126 if self.nextScheduleIsConditional and what=='Path': 01127 self.conditionalPaths.append(prefix) 01128 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, seq) )) 01129 self.schedule.append(getattr(self.process,prefix)) 01130 else: 01131 for i,s in enumerate(seq.split('+')): 01132 sn=prefix+'%d'%(i) 01133 setattr(self.process,sn,getattr(cms,what)( getattr(self.process, s) )) 01134 self.schedule.append(getattr(self.process,sn)) 01135 return
| def ConfigBuilder::ConfigBuilder::scheduleSequenceAtEnd | ( | self, | |
| seq, | |||
| prefix | |||
| ) |
Definition at line 1136 of file ConfigBuilder.py.
ConfigBuilder::ConfigBuilder::_options [private] |
Definition at line 154 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 1732 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
load the relevant part
needs to be in reverse order
print it in the configuration
Definition at line 1460 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
put the filtering path in the schedule
Definition at line 154 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
put it before all the other paths
Definition at line 154 of file ConfigBuilder.py.
adding standard sequences might change the inputEventContent option and therefore needs to be finalized after
Definition at line 1855 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 316 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 808 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
1.7.3