|
|
|
The main building routines
Definition at line 151 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::__init__ | ( | self, | |
| options, | |||
process = None, |
|||
with_output = False, |
|||
with_input = False |
|||
| ) |
options taken from old cmsDriver and optparse
Definition at line 154 of file ConfigBuilder.py.
00155 : 00156 """options taken from old cmsDriver and optparse """ 00157 00158 options.outfile_name = options.dirout+options.fileout 00159 00160 self._options = options 00161 00162 if self._options.isData and options.isMC: 00163 raise Exception("ERROR: You may specify only --data or --mc, not both") 00164 #if not self._options.conditions: 00165 # raise Exception("ERROR: No conditions given!\nPlease specify conditions. E.g. via --conditions=IDEAL_30X::All") 00166 00167 if hasattr(self._options,"datatier") and self._options.datatier and 'DQMROOT' in self._options.datatier and 'ENDJOB' in self._options.step: 00168 self._options.step=self._options.step.replace(',ENDJOB','') 00169 00170 # what steps are provided by this class? 00171 stepList = [re.sub(r'^prepare_', '', methodName) for methodName in ConfigBuilder.__dict__ if methodName.startswith('prepare_')] 00172 self.stepMap={} 00173 for step in self._options.step.split(","): 00174 if step=='': continue 00175 stepParts = step.split(":") 00176 stepName = stepParts[0] 00177 if stepName not in stepList and not stepName.startswith('re'): 00178 raise ValueError("Step "+stepName+" unknown") 00179 if len(stepParts)==1: 00180 self.stepMap[stepName]="" 00181 elif len(stepParts)==2: 00182 self.stepMap[stepName]=stepParts[1].split('+') 00183 elif len(stepParts)==3: 00184 self.stepMap[stepName]=(stepParts[2].split('+'),stepParts[1]) 00185 else: 00186 raise ValueError("Step definition "+step+" invalid") 00187 #print "map of steps is:",self.stepMap 00188 00189 self.with_output = with_output 00190 if hasattr(self._options,"no_output_flag") and self._options.no_output_flag: 00191 self.with_output = False 00192 self.with_input = with_input 00193 if process == None: 00194 self.process = cms.Process(self._options.name) 00195 else: 00196 self.process = process 00197 self.imports = [] 00198 self.define_Configs() 00199 self.schedule = list() 00200 00201 # we are doing three things here: 00202 # creating a process to catch errors 00203 # building the code to re-create the process 00204 00205 self.additionalCommands = [] 00206 # TODO: maybe a list of to be dumped objects would help as well 00207 self.blacklist_paths = [] 00208 self.addedObjects = [] 00209 self.additionalOutputs = {} 00210 00211 self.productionFilterSequence = None 00212 self.nextScheduleIsConditional=False 00213 self.conditionalPaths=[]
| def ConfigBuilder::ConfigBuilder::addCommon | ( | self | ) |
Definition at line 278 of file ConfigBuilder.py.
00279 : 00280 if 'HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys(): 00281 self.process.options = cms.untracked.PSet( Rethrow = cms.untracked.vstring('ProductNotFound'),fileMode = cms.untracked.string('FULLMERGE')) 00282 else: 00283 self.process.options = cms.untracked.PSet( ) 00284 self.addedObjects.append(("","options")) 00285 00286 if self._options.lazy_download: 00287 self.process.AdaptorConfig = cms.Service("AdaptorConfig", 00288 stats = cms.untracked.bool(True), 00289 enable = cms.untracked.bool(True), 00290 cacheHint = cms.untracked.string("lazy-download"), 00291 readHint = cms.untracked.string("read-ahead-buffered") 00292 ) 00293 self.addedObjects.append(("Setup lazy download","AdaptorConfig")) 00294 00295 #self.process.cmsDriverCommand = cms.untracked.PSet( command=cms.untracked.string('cmsDriver.py '+self._options.arguments) ) 00296 #self.addedObjects.append(("what cmsDriver command was used","cmsDriverCommand")) 00297 00298 if self._options.profile: 00299 (start, interval, eventFormat, jobFormat)=self.profileOptions() 00300 self.process.IgProfService = cms.Service("IgProfService", 00301 reportFirstEvent = cms.untracked.int32(start), 00302 reportEventInterval = cms.untracked.int32(interval), 00303 reportToFileAtPostEvent = cms.untracked.string("| gzip -c > %s"%(eventFormat)), 00304 reportToFileAtPostEndJob = cms.untracked.string("| gzip -c > %s"%(jobFormat))) 00305 self.addedObjects.append(("Setup IGProf Service for profiling","IgProfService"))
| def ConfigBuilder::ConfigBuilder::addConditions | ( | self | ) |
Add conditions to the process
Definition at line 697 of file ConfigBuilder.py.
00698 : 00699 """Add conditions to the process""" 00700 if not self._options.conditions: return 00701 00702 if 'FrontierConditions_GlobalTag' in self._options.conditions: 00703 print 'using FrontierConditions_GlobalTag in --conditions is not necessary anymore and will be deprecated soon. please update your command line' 00704 self._options.conditions = self._options.conditions.replace("FrontierConditions_GlobalTag,",'') 00705 00706 self.loadAndRemember(self.ConditionsDefaultCFF) 00707 00708 from Configuration.AlCa.GlobalTag import GlobalTag 00709 self.process.GlobalTag = GlobalTag(self.process.GlobalTag, self._options.conditions, self._options.custom_conditions) 00710 self.additionalCommands.append('from Configuration.AlCa.GlobalTag import GlobalTag') 00711 self.additionalCommands.append('process.GlobalTag = GlobalTag(process.GlobalTag, %s, %s)' % (repr(self._options.conditions), repr(self._options.custom_conditions))) 00712 00713 if self._options.slhc: 00714 self.loadAndRemember("SLHCUpgradeSimulations/Geometry/fakeConditions_%s_cff"%(self._options.slhc,)) 00715
| def ConfigBuilder::ConfigBuilder::addCustomise | ( | self | ) |
Include the customise code
Definition at line 716 of file ConfigBuilder.py.
00717 : 00718 """Include the customise code """ 00719 00720 custOpt=self._options.customisation_file.split(",") 00721 custMap={} 00722 for opt in custOpt: 00723 if opt=='': continue 00724 if opt.count('.')>1: 00725 raise Exception("more than . in the specification:"+opt) 00726 fileName=opt.split('.')[0] 00727 if opt.count('.')==0: rest='customise' 00728 else: 00729 rest=opt.split('.')[1] 00730 if rest=='py': rest='customise' #catch the case of --customise file.py 00731 00732 if fileName in custMap: 00733 custMap[fileName].extend(rest.split('+')) 00734 else: 00735 custMap[fileName]=rest.split('+') 00736 00737 if len(custMap)==0: 00738 final_snippet='\n' 00739 else: 00740 final_snippet='\n# customisation of the process.\n' 00741 00742 allFcn=[] 00743 for opt in custMap: 00744 allFcn.extend(custMap[opt]) 00745 for fcn in allFcn: 00746 if allFcn.count(fcn)!=1: 00747 raise Exception("cannot specify twice "+fcn+" as a customisation method") 00748 00749 for f in custMap: 00750 # let python search for that package and do syntax checking at the same time 00751 packageName = f.replace(".py","").replace("/",".") 00752 __import__(packageName) 00753 package = sys.modules[packageName] 00754 00755 # now ask the package for its definition and pick .py instead of .pyc 00756 customiseFile = re.sub(r'\.pyc$', '.py', package.__file__) 00757 00758 final_snippet+='\n# Automatic addition of the customisation function from '+packageName+'\n' 00759 if self._options.inline_custom: 00760 for line in file(customiseFile,'r'): 00761 if "import FWCore.ParameterSet.Config" in line: 00762 continue 00763 final_snippet += line 00764 else: 00765 final_snippet += 'from %s import %s \n'%(packageName,','.join(custMap[f])) 00766 for fcn in custMap[f]: 00767 print "customising the process with",fcn,"from",f 00768 if not hasattr(package,fcn): 00769 #bound to fail at run time 00770 raise Exception("config "+f+" has no function "+fcn) 00771 #execute the command 00772 self.process=getattr(package,fcn)(self.process) 00773 #and print it in the configuration 00774 final_snippet += "\n#call to customisation function "+fcn+" imported from "+packageName 00775 final_snippet += "\nprocess = %s(process)\n"%(fcn,) 00776 00777 if len(custMap)!=0: 00778 final_snippet += '\n# End of customisation functions\n' 00779 00780 ### now for a usuful command 00781 if self._options.customise_commands: 00782 import string 00783 final_snippet +='\n# Customisation from command line' 00784 for com in self._options.customise_commands.split('\\n'): 00785 com=string.lstrip(com) 00786 self.executeAndRemember(com) 00787 final_snippet +='\n'+com 00788 00789 return final_snippet
| def ConfigBuilder::ConfigBuilder::addExtraStream | ( | self, | |
| name, | |||
| stream, | |||
workflow = 'full' |
|||
| ) |
Definition at line 1018 of file ConfigBuilder.py.
01019 : 01020 # define output module and go from there 01021 output = cms.OutputModule("PoolOutputModule") 01022 if stream.selectEvents.parameters_().__len__()!=0: 01023 output.SelectEvents = stream.selectEvents 01024 else: 01025 output.SelectEvents = cms.untracked.PSet() 01026 output.SelectEvents.SelectEvents=cms.vstring() 01027 if isinstance(stream.paths,tuple): 01028 for path in stream.paths: 01029 output.SelectEvents.SelectEvents.append(path.label()) 01030 else: 01031 output.SelectEvents.SelectEvents.append(stream.paths.label()) 01032 01033 01034 01035 if isinstance(stream.content,str): 01036 evtPset=getattr(self.process,stream.content) 01037 for p in evtPset.parameters_(): 01038 setattr(output,p,getattr(evtPset,p)) 01039 if not self._options.inlineEventContent: 01040 def doNotInlineEventContent(instance,label = "process."+stream.content+".outputCommands"): 01041 return label 01042 output.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 01043 else: 01044 output.outputCommands = stream.content 01045 01046 01047 output.fileName = cms.untracked.string(self._options.dirout+stream.name+'.root') 01048 01049 output.dataset = cms.untracked.PSet( dataTier = stream.dataTier, 01050 filterName = cms.untracked.string(stream.name)) 01051 01052 if self._options.filtername: 01053 output.dataset.filterName= cms.untracked.string(self._options.filtername+"_"+stream.name) 01054 01055 #add an automatic flushing to limit memory consumption 01056 output.eventAutoFlushCompressedSize=cms.untracked.int32(5*1024*1024) 01057 01058 if workflow in ("producers,full"): 01059 if isinstance(stream.paths,tuple): 01060 for path in stream.paths: 01061 self.schedule.append(path) 01062 else: 01063 self.schedule.append(stream.paths) 01064 01065 01066 # in case of relvals we don't want to have additional outputs 01067 if (not self._options.relval) and workflow in ("full","output"): 01068 self.additionalOutputs[name] = output 01069 setattr(self.process,name,output) 01070 01071 if workflow == 'output': 01072 # adjust the select events to the proper trigger results from previous process 01073 filterList = output.SelectEvents.SelectEvents 01074 for i, filter in enumerate(filterList): 01075 filterList[i] = filter+":"+self._options.triggerResultsProcess 01076 01077 return output
| def ConfigBuilder::ConfigBuilder::addMaxEvents | ( | self | ) |
Here we decide how many evts will be processed
Definition at line 306 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::addOutput | ( | self | ) |
Add output module to the process
Definition at line 440 of file ConfigBuilder.py.
00441 : 00442 """ Add output module to the process """ 00443 result="" 00444 if self._options.outputDefinition: 00445 if self._options.datatier: 00446 print "--datatier & --eventcontent options ignored" 00447 00448 def anyOf(listOfKeys,dict,opt=None): 00449 for k in listOfKeys: 00450 if k in dict: 00451 toReturn=dict[k] 00452 dict.pop(k) 00453 return toReturn 00454 if opt!=None: 00455 return opt 00456 else: 00457 raise Exception("any of "+','.join(listOfKeys)+" are mandatory entries of --output options") 00458 00459 #new output convention with a list of dict 00460 outList = eval(self._options.outputDefinition) 00461 for (id,outDefDict) in enumerate(outList): 00462 outDefDictStr=outDefDict.__str__() 00463 if not isinstance(outDefDict,dict): 00464 raise Exception("--output needs to be passed a list of dict"+self._options.outputDefinition+" is invalid") 00465 #requires option: tier 00466 theTier=anyOf(['t','tier','dataTier'],outDefDict) 00467 #optional option: eventcontent, filtername, selectEvents, moduleLabel, filename 00468 ## event content 00469 theStreamType=anyOf(['e','ec','eventContent','streamType'],outDefDict,theTier) 00470 theFilterName=anyOf(['f','ftN','filterName'],outDefDict,'') 00471 theSelectEvent=anyOf(['s','sE','selectEvents'],outDefDict,'') 00472 theModuleLabel=anyOf(['l','mL','moduleLabel'],outDefDict,'') 00473 theExtraOutputCommands=anyOf(['o','oC','outputCommands'],outDefDict,'') 00474 # module label has a particular role 00475 if not theModuleLabel: 00476 tryNames=[theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+'output', 00477 theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+'output', 00478 theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+theSelectEvent.split(',')[0].replace(':','for').replace(' ','')+'output' 00479 ] 00480 for name in tryNames: 00481 if not hasattr(self.process,name): 00482 theModuleLabel=name 00483 break 00484 if not theModuleLabel: 00485 raise Exception("cannot find a module label for specification: "+outDefDictStr) 00486 if id==0: 00487 defaultFileName=self._options.outfile_name 00488 else: 00489 defaultFileName=self._options.outfile_name.replace('.root','_in'+theTier+'.root') 00490 00491 theFileName=self._options.dirout+anyOf(['fn','fileName'],outDefDict,defaultFileName) 00492 if not theFileName.endswith('.root'): 00493 theFileName+='.root' 00494 00495 if len(outDefDict.keys()): 00496 raise Exception("unused keys from --output options: "+','.join(outDefDict.keys())) 00497 if theStreamType=='DQMROOT': theStreamType='DQM' 00498 if theStreamType=='ALL': 00499 theEventContent = cms.PSet(outputCommands = cms.untracked.vstring('keep *')) 00500 else: 00501 theEventContent = getattr(self.process, theStreamType+"EventContent") 00502 00503 if theStreamType=='ALCARECO' and not theFilterName: 00504 theFilterName='StreamALCACombined' 00505 00506 CppType='PoolOutputModule' 00507 if theStreamType=='DQM' and theTier=='DQMROOT': CppType='DQMRootOutputModule' 00508 output = cms.OutputModule(CppType, 00509 theEventContent.clone(), 00510 fileName = cms.untracked.string(theFileName), 00511 dataset = cms.untracked.PSet( 00512 dataTier = cms.untracked.string(theTier), 00513 filterName = cms.untracked.string(theFilterName)) 00514 ) 00515 if not theSelectEvent and hasattr(self.process,'generation_step'): 00516 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step')) 00517 if not theSelectEvent and hasattr(self.process,'filtering_step'): 00518 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step')) 00519 if theSelectEvent: 00520 output.SelectEvents =cms.untracked.PSet(SelectEvents = cms.vstring(theSelectEvent)) 00521 00522 if hasattr(self.process,theModuleLabel): 00523 raise Exception("the current process already has a module "+theModuleLabel+" defined") 00524 #print "creating output module ",theModuleLabel 00525 setattr(self.process,theModuleLabel,output) 00526 outputModule=getattr(self.process,theModuleLabel) 00527 setattr(self.process,theModuleLabel+'_step',cms.EndPath(outputModule)) 00528 path=getattr(self.process,theModuleLabel+'_step') 00529 self.schedule.append(path) 00530 00531 if not self._options.inlineEventContent and hasattr(self.process,theStreamType+"EventContent"): 00532 def doNotInlineEventContent(instance,label = "cms.untracked.vstring(process."+theStreamType+"EventContent.outputCommands)"): 00533 return label 00534 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 00535 if theExtraOutputCommands: 00536 if not isinstance(theExtraOutputCommands,list): 00537 raise Exception("extra ouput command in --option must be a list of strings") 00538 if hasattr(self.process,theStreamType+"EventContent"): 00539 self.executeAndRemember('process.%s.outputCommands.extend(%s)'%(theModuleLabel,theExtraOutputCommands)) 00540 else: 00541 outputModule.outputCommands.extend(theExtraOutputCommands) 00542 00543 result+="\nprocess."+theModuleLabel+" = "+outputModule.dumpPython() 00544 00545 ##ends the --output options model 00546 return result 00547 00548 streamTypes=self._options.eventcontent.split(',') 00549 tiers=self._options.datatier.split(',') 00550 if not self._options.outputDefinition and len(streamTypes)!=len(tiers): 00551 raise Exception("number of event content arguments does not match number of datatier arguments") 00552 00553 # if the only step is alca we don't need to put in an output 00554 if self._options.step.split(',')[0].split(':')[0] == 'ALCA': 00555 return "\n" 00556 00557 for i,(streamType,tier) in enumerate(zip(streamTypes,tiers)): 00558 if streamType=='': continue 00559 if streamType=='DQMROOT': streamType='DQM' 00560 theEventContent = getattr(self.process, streamType+"EventContent") 00561 if i==0: 00562 theFileName=self._options.outfile_name 00563 theFilterName=self._options.filtername 00564 else: 00565 theFileName=self._options.outfile_name.replace('.root','_in'+streamType+'.root') 00566 theFilterName=self._options.filtername 00567 CppType='PoolOutputModule' 00568 if streamType=='DQM' and tier=='DQMROOT': CppType='DQMRootOutputModule' 00569 output = cms.OutputModule(CppType, 00570 theEventContent, 00571 fileName = cms.untracked.string(theFileName), 00572 dataset = cms.untracked.PSet(dataTier = cms.untracked.string(tier), 00573 filterName = cms.untracked.string(theFilterName) 00574 ) 00575 ) 00576 if hasattr(self.process,"generation_step"): 00577 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step')) 00578 if hasattr(self.process,"filtering_step"): 00579 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('filtering_step')) 00580 00581 if streamType=='ALCARECO': 00582 output.dataset.filterName = cms.untracked.string('StreamALCACombined') 00583 00584 outputModuleName=streamType+'output' 00585 setattr(self.process,outputModuleName,output) 00586 outputModule=getattr(self.process,outputModuleName) 00587 setattr(self.process,outputModuleName+'_step',cms.EndPath(outputModule)) 00588 path=getattr(self.process,outputModuleName+'_step') 00589 self.schedule.append(path) 00590 00591 if not self._options.inlineEventContent: 00592 def doNotInlineEventContent(instance,label = "process."+streamType+"EventContent.outputCommands"): 00593 return label 00594 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 00595 00596 result+="\nprocess."+outputModuleName+" = "+outputModule.dumpPython() 00597 00598 return result
| def ConfigBuilder::ConfigBuilder::addSource | ( | self | ) |
Here the source is built. Priority: file, generator
Definition at line 311 of file ConfigBuilder.py.
00312 : 00313 """Here the source is built. Priority: file, generator""" 00314 self.addedObjects.append(("Input source","source")) 00315 00316 def filesFromOption(self): 00317 for entry in self._options.filein.split(','): 00318 print "entry",entry 00319 if entry.startswith("filelist:"): 00320 filesFromList(entry[9:],self.process.source) 00321 elif entry.startswith("dbs:"): 00322 filesFromDBSQuery('find file where dataset = %s'%(entry[4:]),self.process.source) 00323 else: 00324 self.process.source.fileNames.append(self._options.dirin+entry) 00325 if self._options.secondfilein: 00326 if not hasattr(self.process.source,"secondaryFileNames"): 00327 raise Exception("--secondfilein not compatible with "+self._options.filetype+"input type") 00328 for entry in self._options.secondfilein.split(','): 00329 print "entry",entry 00330 if entry.startswith("filelist:"): 00331 self.process.source.secondaryFileNames.extend((filesFromList(entry[9:]))[0]) 00332 elif entry.startswith("dbs:"): 00333 self.process.source.secondaryFileNames.extend((filesFromDBSQuery('find file where dataset = %s'%(entry[4:])))[0]) 00334 else: 00335 self.process.source.secondaryFileNames.append(self._options.dirin+entry) 00336 00337 if self._options.filein or self._options.dbsquery: 00338 if self._options.filetype == "EDM": 00339 self.process.source=cms.Source("PoolSource", 00340 fileNames = cms.untracked.vstring(), 00341 secondaryFileNames= cms.untracked.vstring()) 00342 filesFromOption(self) 00343 elif self._options.filetype == "DAT": 00344 self.process.source=cms.Source("NewEventStreamFileReader",fileNames = cms.untracked.vstring()) 00345 filesFromOption(self) 00346 elif self._options.filetype == "LHE": 00347 self.process.source=cms.Source("LHESource", fileNames = cms.untracked.vstring()) 00348 if self._options.filein.startswith("lhe:"): 00349 #list the article directory automatically 00350 args=self._options.filein.split(':') 00351 article=args[1] 00352 print 'LHE input from article ',article 00353 location='/store/lhe/' 00354 import os 00355 textOfFiles=os.popen('cmsLHEtoEOSManager.py -l '+article) 00356 for line in textOfFiles: 00357 for fileName in [x for x in line.split() if '.lhe' in x]: 00358 self.process.source.fileNames.append(location+article+'/'+fileName) 00359 if len(args)>2: 00360 self.process.source.skipEvents = cms.untracked.uint32(int(args[2])) 00361 else: 00362 filesFromOption(self) 00363 00364 00365 elif self._options.filetype == "DQM": 00366 self.process.source=cms.Source("DQMRootSource", 00367 fileNames = cms.untracked.vstring()) 00368 filesFromOption(self) 00369 00370 if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"): 00371 self.process.source.processingMode = cms.untracked.string("RunsAndLumis") 00372 00373 if self._options.dbsquery!='': 00374 self.process.source=cms.Source("PoolSource", fileNames = cms.untracked.vstring(),secondaryFileNames = cms.untracked.vstring()) 00375 filesFromDBSQuery(self._options.dbsquery,self.process.source) 00376 00377 if self._options.inputEventContent: 00378 import copy 00379 def dropSecondDropStar(iec): 00380 #drop occurence of 'drop *' in the list 00381 count=0 00382 for item in iec: 00383 if item=='drop *': 00384 if count!=0: 00385 iec.remove(item) 00386 count+=1 00387 00388 00389 ## allow comma separated input eventcontent 00390 if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring() 00391 for evct in self._options.inputEventContent.split(','): 00392 if evct=='': continue 00393 theEventContent = getattr(self.process, evct+"EventContent") 00394 if hasattr(theEventContent,'outputCommands'): 00395 self.process.source.inputCommands.extend(copy.copy(theEventContent.outputCommands)) 00396 if hasattr(theEventContent,'inputCommands'): 00397 self.process.source.inputCommands.extend(copy.copy(theEventContent.inputCommands)) 00398 00399 dropSecondDropStar(self.process.source.inputCommands) 00400 00401 if not self._options.dropDescendant: 00402 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False) 00403 00404 if self._options.inputCommands: 00405 if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring() 00406 for command in self._options.inputCommands.split(','): 00407 # remove whitespace around the keep/drop statements 00408 command = command.strip() 00409 if command=='': continue 00410 self.process.source.inputCommands.append(command) 00411 if not self._options.dropDescendant: 00412 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False) 00413 00414 if self._options.lumiToProcess: 00415 import FWCore.PythonUtilities.LumiList as LumiList 00416 self.process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange( LumiList.LumiList(self._options.lumiToProcess).getCMSSWString().split(',') ) 00417 00418 if 'GEN' in self.stepMap or 'LHE' in self.stepMap or (not self._options.filein and hasattr(self._options, "evt_type")): 00419 if self.process.source is None: 00420 self.process.source=cms.Source("EmptySource") 00421 # if option himix is active, drop possibly duplicate DIGI-RAW info: 00422 if self._options.himix==True: 00423 self.process.source.inputCommands = cms.untracked.vstring('drop *','keep *_generator_*_*','keep *_g4SimHits_*_*') 00424 self.process.source.dropDescendantsOfDroppedBranches=cms.untracked.bool(False) 00425 # modify source in case of run-dependent MC 00426 if self._options.runsAndWeightsForMC != None or self._options.runsScenarioForMC != None : 00427 # setting run numbers is allowed only for MC 00428 if not self._options.isMC : 00429 raise Exception("options --runsAndWeightsForMC and --runsScenarioForMC are only valid for MC") 00430 self.additionalCommands.append('import SimGeneral.Configuration.ThrowAndSetRandomRun as ThrowAndSetRandomRun') 00431 if self._options.runsAndWeightsForMC != None : 00432 # ThrowAndSetRandomRun throws a pseudo-random run according to runsAndWeightsForMC specified in input 00433 self.additionalCommands.append('ThrowAndSetRandomRun.throwAndSetRandomRun(process.source,'+str(self._options.runsAndWeightsForMC)+')') 00434 elif self._options.runsScenarioForMC != None : 00435 # ThrowAndSetRandomRun throws a pseudo-random run according to an existing run scenario 00436 from Configuration.StandardSequences.RunsAndWeights import RunsAndWeights 00437 self.additionalCommands.append('import '+RunsAndWeights[self._options.runsScenarioForMC]+' as RunScenario') 00438 self.additionalCommands.append('ThrowAndSetRandomRun.throwAndSetRandomRun(process.source,RunScenario.runProbabilityDistribution)') 00439 return
| def ConfigBuilder::ConfigBuilder::addStandardSequences | ( | self | ) |
Add selected standard sequences to the process
Definition at line 599 of file ConfigBuilder.py.
00600 : 00601 """ 00602 Add selected standard sequences to the process 00603 """ 00604 # load the pile up file 00605 if self._options.pileup: 00606 pileupSpec=self._options.pileup.split(',')[0] 00607 from Configuration.StandardSequences.Mixing import Mixing,defineMixing 00608 if not pileupSpec in Mixing and '.' not in pileupSpec and 'file:' not in pileupSpec: 00609 raise Exception(pileupSpec+' is not a know mixing scenario:\n available are: '+'\n'.join(Mixing.keys())) 00610 if '.' in pileupSpec: 00611 mixingDict={'file':pileupSpec} 00612 elif pileupSpec.startswith('file:'): 00613 mixingDict={'file':pileupSpec[5:]} 00614 else: 00615 import copy 00616 mixingDict=copy.copy(Mixing[pileupSpec]) 00617 if len(self._options.pileup.split(','))>1: 00618 mixingDict.update(eval(self._options.pileup[self._options.pileup.find(',')+1:])) 00619 if 'file:' in pileupSpec: 00620 #the file is local 00621 self.process.load(mixingDict['file']) 00622 print "inlining mixing module configuration" 00623 self._options.inlineObjets+=',mix' 00624 else: 00625 self.loadAndRemember(mixingDict['file']) 00626 00627 mixingDict.pop('file') 00628 if self._options.pileup_input: 00629 if self._options.pileup_input.startswith('dbs'): 00630 mixingDict['F']=filesFromDBSQuery('find file where dataset = %s'%(self._options.pileup_input[4:],))[0] 00631 else: 00632 mixingDict['F']=self._options.pileup_input.split(',') 00633 specialization=defineMixing(mixingDict,'FASTSIM' in self.stepMap) 00634 for command in specialization: 00635 self.executeAndRemember(command) 00636 if len(mixingDict)!=0: 00637 raise Exception('unused mixing specification: '+mixingDict.keys().__str__()) 00638 00639 00640 # load the geometry file 00641 try: 00642 if len(self.stepMap): 00643 self.loadAndRemember(self.GeometryCFF) 00644 if 'SIM' in self.stepMap or 'reSIM' in self.stepMap: 00645 self.loadAndRemember(self.SimGeometryCFF) 00646 if self.geometryDBLabel: 00647 self.executeAndRemember('process.XMLFromDBSource.label = cms.string("%s")'%(self.geometryDBLabel)) 00648 except ImportError: 00649 print "Geometry option",self._options.geometry,"unknown." 00650 raise 00651 00652 if len(self.stepMap): 00653 self.loadAndRemember(self.magFieldCFF) 00654 00655 # what steps are provided by this class? 00656 stepList = [re.sub(r'^prepare_', '', methodName) for methodName in ConfigBuilder.__dict__ if methodName.startswith('prepare_')] 00657 00658 ### Benedikt can we add here a check that assure that we are going to generate a correct config file? 00659 ### i.e. the harvesting do not have to include other step...... 00660 00661 # look which steps are requested and invoke the corresponding method 00662 for step in self._options.step.split(","): 00663 if step == "": 00664 continue 00665 print step 00666 if step.startswith('re'): 00667 ##add the corresponding input content 00668 if step[2:] not in self._options.donotDropOnInput: 00669 self._options.inputEventContent='%s,%s'%(step.split(":")[0].upper(),self._options.inputEventContent) 00670 step=step[2:] 00671 stepParts = step.split(":") # for format STEP:alternativeSequence 00672 stepName = stepParts[0] 00673 if stepName not in stepList: 00674 raise ValueError("Step "+stepName+" unknown") 00675 if len(stepParts)==1: 00676 getattr(self,"prepare_"+step)(sequence = getattr(self,step+"DefaultSeq")) 00677 elif len(stepParts)==2: 00678 getattr(self,"prepare_"+stepName)(sequence = stepParts[1]) 00679 elif len(stepParts)==3: 00680 getattr(self,"prepare_"+stepName)(sequence = stepParts[1]+','+stepParts[2]) 00681 00682 else: 00683 raise ValueError("Step definition "+step+" invalid") 00684 00685 if self._options.restoreRNDSeeds!=False: 00686 #it is either True, or a process name 00687 if self._options.restoreRNDSeeds==True: 00688 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateLabel=cms.untracked.string("randomEngineStateProducer")') 00689 else: 00690 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateTag=cms.untracked.InputTag("randomEngineStateProducer","","%s")'%(self._options.restoreRNDSeeds)) 00691 if self._options.inputEventContent or self._options.inputCommands: 00692 if self._options.inputCommands: 00693 self._options.inputCommands+='keep *_randomEngineStateProducer_*_*,' 00694 else: 00695 self._options.inputCommands='keep *_randomEngineStateProducer_*_*,' 00696
| def ConfigBuilder::ConfigBuilder::build_production_info | ( | self, | |
| evt_type, | |||
| evtnumber | |||
| ) |
Add useful info for the production.
Definition at line 1818 of file ConfigBuilder.py.
01819 : 01820 """ Add useful info for the production. """ 01821 self.process.configurationMetadata=cms.untracked.PSet\ 01822 (version=cms.untracked.string("$Revision: 1.381.2.20 $"), 01823 name=cms.untracked.string("PyReleaseValidation"), 01824 annotation=cms.untracked.string(evt_type+ " nevts:"+str(evtnumber)) 01825 ) 01826 01827 self.addedObjects.append(("Production Info","configurationMetadata")) 01828
| def ConfigBuilder::ConfigBuilder::define_Configs | ( | self | ) |
Definition at line 794 of file ConfigBuilder.py.
00795 : 00796 if len(self.stepMap): 00797 self.loadAndRemember('Configuration/StandardSequences/Services_cff') 00798 if self._options.particleTable not in defaultOptions.particleTableList: 00799 print 'Invalid particle table provided. Options are:' 00800 print defaultOptions.particleTable 00801 sys.exit(-1) 00802 else: 00803 if len(self.stepMap): 00804 self.loadAndRemember('SimGeneral.HepPDTESSource.'+self._options.particleTable+'_cfi') 00805 00806 self.loadAndRemember('FWCore/MessageService/MessageLogger_cfi') 00807 00808 self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreams_cff" 00809 self.GENDefaultCFF="Configuration/StandardSequences/Generator_cff" 00810 self.SIMDefaultCFF="Configuration/StandardSequences/Sim_cff" 00811 self.DIGIDefaultCFF="Configuration/StandardSequences/Digi_cff" 00812 self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRaw_cff" 00813 self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1Emulator_cff' 00814 self.L1MENUDefaultCFF="Configuration/StandardSequences/L1TriggerDefaultMenu_cff" 00815 self.HLTDefaultCFF="Configuration/StandardSequences/HLTtable_cff" 00816 self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_Data_cff" 00817 self.L1RecoDefaultCFF="Configuration/StandardSequences/L1Reco_cff" 00818 self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_Data_cff" 00819 self.SKIMDefaultCFF="Configuration/StandardSequences/Skims_cff" 00820 self.POSTRECODefaultCFF="Configuration/StandardSequences/PostRecoGenerator_cff" 00821 self.VALIDATIONDefaultCFF="Configuration/StandardSequences/Validation_cff" 00822 self.L1HwValDefaultCFF = "Configuration/StandardSequences/L1HwVal_cff" 00823 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOffline_cff" 00824 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/Harvesting_cff" 00825 self.ALCAHARVESTDefaultCFF="Configuration/StandardSequences/AlCaHarvesting_cff" 00826 self.ENDJOBDefaultCFF="Configuration/StandardSequences/EndOfProcess_cff" 00827 self.ConditionsDefaultCFF = "Configuration/StandardSequences/FrontierConditions_GlobalTag_cff" 00828 self.CFWRITERDefaultCFF = "Configuration/StandardSequences/CrossingFrameWriter_cff" 00829 self.REPACKDefaultCFF="Configuration/StandardSequences/DigiToRaw_Repack_cff" 00830 00831 if "DATAMIX" in self.stepMap.keys(): 00832 self.DATAMIXDefaultCFF="Configuration/StandardSequences/DataMixer"+self._options.datamix+"_cff" 00833 self.DIGIDefaultCFF="Configuration/StandardSequences/DigiDM_cff" 00834 self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRawDM_cff" 00835 self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1EmulatorDM_cff' 00836 00837 self.ALCADefaultSeq=None 00838 self.LHEDefaultSeq='externalLHEProducer' 00839 self.GENDefaultSeq='pgen' 00840 self.SIMDefaultSeq=None 00841 self.DIGIDefaultSeq='pdigi' 00842 self.DATAMIXDefaultSeq=None 00843 self.DIGI2RAWDefaultSeq='DigiToRaw' 00844 self.HLTDefaultSeq='GRun' 00845 self.L1DefaultSeq=None 00846 self.L1REPACKDefaultSeq='GT' 00847 self.HARVESTINGDefaultSeq=None 00848 self.ALCAHARVESTDefaultSeq=None 00849 self.CFWRITERDefaultSeq=None 00850 self.RAW2DIGIDefaultSeq='RawToDigi' 00851 self.L1RecoDefaultSeq='L1Reco' 00852 if 'RAW2DIGI' in self.stepMap and 'RECO' in self.stepMap: 00853 self.RECODefaultSeq='reconstruction' 00854 else: 00855 self.RECODefaultSeq='reconstruction_fromRECO' 00856 00857 self.POSTRECODefaultSeq=None 00858 self.L1HwValDefaultSeq='L1HwVal' 00859 self.DQMDefaultSeq='DQMOffline' 00860 self.FASTSIMDefaultSeq='all' 00861 self.VALIDATIONDefaultSeq='' 00862 self.PATLayer0DefaultSeq='all' 00863 self.ENDJOBDefaultSeq='endOfProcess' 00864 self.REPACKDefaultSeq='DigiToRawRepack' 00865 00866 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContent_cff" 00867 00868 # if fastsim switch event content 00869 if "FASTSIM" in self.stepMap.keys(): 00870 self.GENDefaultSeq='pgen_genonly' 00871 self.EVTCONTDefaultCFF = "FastSimulation/Configuration/EventContent_cff" 00872 self.VALIDATIONDefaultCFF = "FastSimulation.Configuration.Validation_cff" 00873 00874 if not self._options.beamspot: 00875 self._options.beamspot=VtxSmearedDefaultKey 00876 00877 # if its MC then change the raw2digi 00878 if self._options.isMC==True: 00879 self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_cff" 00880 self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_cff" 00881 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineMC_cff" 00882 self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreamsMC_cff" 00883 else: 00884 self._options.beamspot = None 00885 00886 #patch for gen, due to backward incompatibility 00887 if 'reGEN' in self.stepMap: 00888 self.GENDefaultSeq='fixGenInfo' 00889 00890 if self._options.scenario=='cosmics': 00891 self.DIGIDefaultCFF="Configuration/StandardSequences/DigiCosmics_cff" 00892 self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionCosmics_cff" 00893 self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsCosmics_cff" 00894 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentCosmics_cff" 00895 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmics_cff" 00896 if self._options.isMC==True: 00897 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmicsMC_cff" 00898 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingCosmics_cff" 00899 self.RECODefaultSeq='reconstructionCosmics' 00900 self.DQMDefaultSeq='DQMOfflineCosmics' 00901 00902 if self._options.himix: 00903 print "From the presence of the himix option, we have determined that this is heavy ions and will use '--scenario HeavyIons'." 00904 self._options.scenario='HeavyIons' 00905 00906 if self._options.scenario=='HeavyIons': 00907 if not self._options.beamspot: 00908 self._options.beamspot=VtxSmearedHIDefaultKey 00909 self.HLTDefaultSeq = 'HIon' 00910 if not self._options.himix: 00911 self.GENDefaultSeq='pgen_hi' 00912 else: 00913 self.GENDefaultSeq='pgen_himix' 00914 self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationHeavyIons_cff" 00915 self.VALIDATIONDefaultSeq='' 00916 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentHeavyIons_cff" 00917 self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionHeavyIons_cff" 00918 self.RECODefaultSeq='reconstructionHeavyIons' 00919 self.ALCADefaultCFF = "Configuration/StandardSequences/AlCaRecoStreamsHeavyIons_cff" 00920 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIons_cff" 00921 self.DQMDefaultSeq='DQMOfflineHeavyIons' 00922 self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsHeavyIons_cff" 00923 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingHeavyIons_cff" 00924 if self._options.isMC==True: 00925 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIonsMC_cff" 00926 00927 00928 self.RAW2RECODefaultSeq=','.join([self.RAW2DIGIDefaultSeq,self.RECODefaultSeq]) 00929 00930 self.USERDefaultSeq='user' 00931 self.USERDefaultCFF=None 00932 00933 # the magnetic field 00934 if self._options.isData: 00935 if self._options.magField==defaultOptions.magField: 00936 print "magnetic field option forced to: AutoFromDBCurrent" 00937 self._options.magField='AutoFromDBCurrent' 00938 self.magFieldCFF = 'Configuration/StandardSequences/MagneticField_'+self._options.magField.replace('.','')+'_cff' 00939 self.magFieldCFF = self.magFieldCFF.replace("__",'_') 00940 00941 # the geometry 00942 self.GeometryCFF='Configuration/StandardSequences/GeometryRecoDB_cff' 00943 self.geometryDBLabel=None 00944 simGeometry='' 00945 if 'FASTSIM' in self.stepMap: 00946 if 'start' in self._options.conditions.lower(): 00947 self.GeometryCFF='FastSimulation/Configuration/Geometries_START_cff' 00948 else: 00949 self.GeometryCFF='FastSimulation/Configuration/Geometries_MC_cff' 00950 else: 00951 def inGeometryKeys(opt): 00952 from Configuration.StandardSequences.GeometryConf import GeometryConf 00953 if opt in GeometryConf: 00954 return GeometryConf[opt] 00955 else: 00956 return opt 00957 00958 geoms=self._options.geometry.split(',') 00959 if len(geoms)==1: geoms=inGeometryKeys(geoms[0]).split(',') 00960 if len(geoms)==2: 00961 #may specify the reco geometry 00962 if '/' in geoms[1] or '_cff' in geoms[1]: 00963 self.GeometryCFF=geoms[1] 00964 else: 00965 self.GeometryCFF='Configuration/StandardSequences/Geometry'+geoms[1]+'_cff' 00966 00967 if (geoms[0].startswith('DB:')): 00968 self.SimGeometryCFF='Configuration/StandardSequences/GeometrySimDB_cff' 00969 self.geometryDBLabel=geoms[0][3:] 00970 print "with DB:" 00971 else: 00972 if '/' in geoms[0] or '_cff' in geoms[0]: 00973 self.SimGeometryCFF=geoms[0] 00974 else: 00975 simGeometry=geoms[0] 00976 if self._options.gflash==True: 00977 self.SimGeometryCFF='Configuration/StandardSequences/Geometry'+geoms[0]+'GFlash_cff' 00978 else: 00979 self.SimGeometryCFF='Configuration/StandardSequences/Geometry'+geoms[0]+'_cff' 00980 00981 # synchronize the geometry configuration and the FullSimulation sequence to be used 00982 if simGeometry not in defaultOptions.geometryExtendedOptions: 00983 self.SIMDefaultCFF="Configuration/StandardSequences/SimIdeal_cff" 00984 00985 if self._options.scenario=='nocoll' or self._options.scenario=='cosmics': 00986 print simGeometry 00987 self.SIMDefaultCFF="Configuration/StandardSequences/SimNOBEAM_cff" 00988 self._options.beamspot='NoSmear' 00989 00990 00991 # Mixing 00992 if self._options.pileup=='default': 00993 from Configuration.StandardSequences.Mixing import MixingDefaultKey,MixingFSDefaultKey 00994 if 'FASTSIM' in self.stepMap: 00995 self._options.pileup=MixingFSDefaultKey 00996 else: 00997 self._options.pileup=MixingDefaultKey 00998 00999 #not driven by a default cff anymore 01000 if self._options.isData: 01001 self._options.pileup=None 01002 if self._options.isMC==True and self._options.himix==False: 01003 if 'FASTSIM' in self.stepMap: 01004 self._options.pileup='FS_'+self._options.pileup 01005 elif self._options.isMC==True and self._options.himix==True: 01006 self._options.pileup='HiMix' 01007 01008 01009 if self._options.slhc: 01010 if 'stdgeom' not in self._options.slhc: 01011 self.SimGeometryCFF='SLHCUpgradeSimulations.Geometry.%s_cmsSimIdealGeometryXML_cff'%(self._options.slhc,) 01012 self.DIGIDefaultCFF='SLHCUpgradeSimulations/Geometry/Digi_%s_cff'%(self._options.slhc,) 01013 if self._options.pileup!=defaultOptions.pileup: 01014 self._options.pileup='SLHC_%s_%s'%(self._options.pileup,self._options.slhc) 01015 01016 self.REDIGIDefaultSeq=self.DIGIDefaultSeq
| def ConfigBuilder::ConfigBuilder::executeAndRemember | ( | self, | |
| command | |||
| ) |
helper routine to remember replace statements
Definition at line 271 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::expandMapping | ( | self, | |
| seqList, | |||
| mapping, | |||
index = None |
|||
| ) |
Definition at line 1655 of file ConfigBuilder.py.
01656 : 01657 maxLevel=20 01658 level=0 01659 while '@' in repr(seqList) and level<maxLevel: 01660 level+=1 01661 for specifiedCommand in seqList: 01662 if specifiedCommand.startswith('@'): 01663 location=specifiedCommand[1:] 01664 if not location in mapping: 01665 raise Exception("Impossible to map "+location+" from "+repr(mapping)) 01666 mappedTo=mapping[location] 01667 if index!=None: 01668 mappedTo=mappedTo[index] 01669 seqList.remove(specifiedCommand) 01670 seqList.extend(mappedTo.split('+')) 01671 break; 01672 if level==maxLevel: 01673 raise Exception("Could not fully expand "+repr(seqList)+" from "+repr(mapping))
| def ConfigBuilder::ConfigBuilder::finalizeFastSimHLT | ( | self | ) |
Definition at line 1769 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::load | ( | self, | |
| includeFile | |||
| ) |
Definition at line 257 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::loadAndRemember | ( | self, | |
| includeFile | |||
| ) |
helper routine to load am memorize imports
Definition at line 262 of file ConfigBuilder.py.
00263 : 00264 """helper routine to load am memorize imports""" 00265 # we could make the imports a on-the-fly data method of the process instance itself 00266 # not sure if the latter is a good idea 00267 includeFile = includeFile.replace('/','.') 00268 self.imports.append(includeFile) 00269 self.process.load(includeFile) 00270 return sys.modules[includeFile]
| def ConfigBuilder::ConfigBuilder::loadDefaultOrSpecifiedCFF | ( | self, | |
| sequence, | |||
| defaultCFF | |||
| ) |
Definition at line 1083 of file ConfigBuilder.py.
01084 : 01085 if ( len(sequence.split('.'))==1 ): 01086 l=self.loadAndRemember(defaultCFF) 01087 elif ( len(sequence.split('.'))==2 ): 01088 l=self.loadAndRemember(sequence.split('.')[0]) 01089 sequence=sequence.split('.')[1] 01090 else: 01091 print "sub sequence configuration must be of the form dir/subdir/cff.a+b+c or cff.a" 01092 print sequence,"not recognized" 01093 raise 01094 return l
| def ConfigBuilder::ConfigBuilder::prepare | ( | self, | |
doChecking = False |
|||
| ) |
Prepare the configuration string and add missing pieces.
Definition at line 1829 of file ConfigBuilder.py.
01830 : 01831 """ Prepare the configuration string and add missing pieces.""" 01832 01833 self.loadAndRemember(self.EVTCONTDefaultCFF) #load the event contents regardless 01834 self.addMaxEvents() 01835 self.addStandardSequences() 01836 if self.with_input: 01837 self.addSource() 01838 self.addConditions() 01839 01840 01841 outputModuleCfgCode="" 01842 if not 'HARVESTING' in self.stepMap.keys() and not 'SKIM' in self.stepMap.keys() and not 'ALCAHARVEST' in self.stepMap.keys() and not 'ALCAOUTPUT' in self.stepMap.keys() and self.with_output: 01843 outputModuleCfgCode=self.addOutput() 01844 01845 self.addCommon() 01846 01847 self.pythonCfgCode = "# Auto generated configuration file\n" 01848 self.pythonCfgCode += "# using: \n# "+__version__[1:-1]+"\n# "+__source__[1:-1]+'\n' 01849 self.pythonCfgCode += "# with command line options: "+self._options.arguments+'\n' 01850 self.pythonCfgCode += "import FWCore.ParameterSet.Config as cms\n\n" 01851 self.pythonCfgCode += "process = cms.Process('"+self.process.name_()+"')\n\n" 01852 01853 self.pythonCfgCode += "# import of standard configurations\n" 01854 for module in self.imports: 01855 self.pythonCfgCode += ("process.load('"+module+"')\n") 01856 01857 # production info 01858 if not hasattr(self.process,"configurationMetadata"): 01859 self.build_production_info(self._options.evt_type, self._options.number) 01860 else: 01861 #the PSet was added via a load 01862 self.addedObjects.append(("Production Info","configurationMetadata")) 01863 01864 self.pythonCfgCode +="\n" 01865 for comment,object in self.addedObjects: 01866 if comment!="": 01867 self.pythonCfgCode += "\n# "+comment+"\n" 01868 self.pythonCfgCode += dumpPython(self.process,object) 01869 01870 # dump the output definition 01871 self.pythonCfgCode += "\n# Output definition\n" 01872 self.pythonCfgCode += outputModuleCfgCode 01873 01874 # dump all additional outputs (e.g. alca or skim streams) 01875 self.pythonCfgCode += "\n# Additional output definition\n" 01876 #I do not understand why the keys are not normally ordered. 01877 nl=self.additionalOutputs.keys() 01878 nl.sort() 01879 for name in nl: 01880 output = self.additionalOutputs[name] 01881 self.pythonCfgCode += "process.%s = %s" %(name, output.dumpPython()) 01882 tmpOut = cms.EndPath(output) 01883 setattr(self.process,name+'OutPath',tmpOut) 01884 self.schedule.append(tmpOut) 01885 01886 # dump all additional commands 01887 self.pythonCfgCode += "\n# Other statements\n" 01888 for command in self.additionalCommands: 01889 self.pythonCfgCode += command + "\n" 01890 01891 #comma separated list of objects that deserve to be inlined in the configuration (typically from a modified config deep down) 01892 for object in self._options.inlineObjets.split(','): 01893 if not object: 01894 continue 01895 if not hasattr(self.process,object): 01896 print 'cannot inline -'+object+'- : not known' 01897 else: 01898 self.pythonCfgCode +='\n' 01899 self.pythonCfgCode +=dumpPython(self.process,object) 01900 01901 # dump all paths 01902 self.pythonCfgCode += "\n# Path and EndPath definitions\n" 01903 for path in self.process.paths: 01904 if getattr(self.process,path) not in self.blacklist_paths: 01905 self.pythonCfgCode += dumpPython(self.process,path) 01906 01907 for endpath in self.process.endpaths: 01908 if getattr(self.process,endpath) not in self.blacklist_paths: 01909 self.pythonCfgCode += dumpPython(self.process,endpath) 01910 01911 # dump the schedule 01912 self.pythonCfgCode += "\n# Schedule definition\n" 01913 result = "process.schedule = cms.Schedule(" 01914 01915 # handling of the schedule 01916 self.process.schedule = cms.Schedule() 01917 for item in self.schedule: 01918 if not isinstance(item, cms.Schedule): 01919 self.process.schedule.append(item) 01920 else: 01921 self.process.schedule.extend(item) 01922 01923 if hasattr(self.process,"HLTSchedule"): 01924 beforeHLT = self.schedule[:self.schedule.index(self.process.HLTSchedule)] 01925 afterHLT = self.schedule[self.schedule.index(self.process.HLTSchedule)+1:] 01926 pathNames = ['process.'+p.label_() for p in beforeHLT] 01927 result += ','.join(pathNames)+')\n' 01928 result += 'process.schedule.extend(process.HLTSchedule)\n' 01929 pathNames = ['process.'+p.label_() for p in afterHLT] 01930 result += 'process.schedule.extend(['+','.join(pathNames)+'])\n' 01931 else: 01932 pathNames = ['process.'+p.label_() for p in self.schedule] 01933 result ='process.schedule = cms.Schedule('+','.join(pathNames)+')\n' 01934 01935 self.pythonCfgCode += result 01936 01937 #repacked version 01938 if self._options.isRepacked: 01939 self.pythonCfgCode +="\n" 01940 self.pythonCfgCode +="from Configuration.PyReleaseValidation.ConfigBuilder import MassReplaceInputTag\n" 01941 self.pythonCfgCode +="MassReplaceInputTag(process)\n" 01942 MassReplaceInputTag(self.process) 01943 01944 # special treatment in case of production filter sequence 2/2 01945 if self.productionFilterSequence: 01946 self.pythonCfgCode +='# filter all path with the production filter sequence\n' 01947 self.pythonCfgCode +='for path in process.paths:\n' 01948 if len(self.conditionalPaths): 01949 self.pythonCfgCode +='\tif not path in %s: continue\n'%str(self.conditionalPaths) 01950 self.pythonCfgCode +='\tgetattr(process,path)._seq = process.%s * getattr(process,path)._seq \n'%(self.productionFilterSequence,) 01951 pfs = getattr(self.process,self.productionFilterSequence) 01952 for path in self.process.paths: 01953 if not path in self.conditionalPaths: continue 01954 getattr(self.process,path)._seq = pfs * getattr(self.process,path)._seq 01955 01956 # dump customise fragment 01957 self.pythonCfgCode += self.addCustomise() 01958 01959 # make the .io file 01960 01961 if self._options.io: 01962 #io=open(self._options.python_filename.replace('.py','.io'),'w') 01963 if not self._options.io.endswith('.io'): self._option.io+='.io' 01964 io=open(self._options.io,'w') 01965 ioJson={} 01966 if hasattr(self.process.source,"fileNames"): 01967 if len(self.process.source.fileNames.value()): 01968 ioJson['primary']=self.process.source.fileNames.value() 01969 if hasattr(self.process.source,"secondaryFileNames"): 01970 if len(self.process.source.secondaryFileNames.value()): 01971 ioJson['secondary']=self.process.source.secondaryFileNames.value() 01972 01973 for (o,om) in self.process.outputModules_().items(): 01974 ioJson[o]=om.fileName.value() 01975 #ioJson['GT']=self._options.conditions 01976 ioJson['GT']=self.process.GlobalTag.globaltag.value() 01977 if self.productionFilterSequence: 01978 ioJson['filter']=self.productionFilterSequence 01979 import json 01980 io.write(json.dumps(ioJson)) 01981 return 01982 01983 01984
| def ConfigBuilder::ConfigBuilder::prepare_ALCA | ( | self, | |
sequence = None, |
|||
workflow = 'full' |
|||
| ) |
Enrich the process with alca streams
Definition at line 1132 of file ConfigBuilder.py.
01133 : 01134 """ Enrich the process with alca streams """ 01135 alcaConfig=self.loadDefaultOrSpecifiedCFF(sequence,self.ALCADefaultCFF) 01136 sequence = sequence.split('.')[-1] 01137 01138 # decide which ALCA paths to use 01139 alcaList = sequence.split("+") 01140 maxLevel=0 01141 from Configuration.AlCa.autoAlca import autoAlca 01142 # support @X from autoAlca.py, and recursion support: i.e T0:@Mu+@EG+... 01143 self.expandMapping(alcaList,autoAlca) 01144 01145 for name in alcaConfig.__dict__: 01146 alcastream = getattr(alcaConfig,name) 01147 shortName = name.replace('ALCARECOStream','') 01148 if shortName in alcaList and isinstance(alcastream,cms.FilteredStream): 01149 output = self.addExtraStream(name,alcastream, workflow = workflow) 01150 if 'DQM' in alcaList: 01151 if not self._options.inlineEventContent and hasattr(self.process,name): 01152 self.executeAndRemember('process.' + name + '.outputCommands.append("keep *_MEtoEDMConverter_*_*")') 01153 else: 01154 output.outputCommands.append("keep *_MEtoEDMConverter_*_*") 01155 01156 #rename the HLT process name in the alca modules 01157 if self._options.hltProcess or 'HLT' in self.stepMap: 01158 if isinstance(alcastream.paths,tuple): 01159 for path in alcastream.paths: 01160 self.renameHLTprocessInSequence(path.label()) 01161 else: 01162 self.renameHLTprocessInSequence(alcastream.paths.label()) 01163 01164 for i in range(alcaList.count(shortName)): 01165 alcaList.remove(shortName) 01166 01167 # DQM needs a special handling 01168 elif name == 'pathALCARECODQM' and 'DQM' in alcaList: 01169 path = getattr(alcaConfig,name) 01170 self.schedule.append(path) 01171 alcaList.remove('DQM') 01172 01173 if isinstance(alcastream,cms.Path): 01174 #black list the alca path so that they do not appear in the cfg 01175 self.blacklist_paths.append(alcastream) 01176 01177 01178 if len(alcaList) != 0: 01179 available=[] 01180 for name in alcaConfig.__dict__: 01181 alcastream = getattr(alcaConfig,name) 01182 if isinstance(alcastream,cms.FilteredStream): 01183 available.append(name.replace('ALCARECOStream','')) 01184 print "The following alcas could not be found "+str(alcaList) 01185 print "available ",available 01186 #print "verify your configuration, ignoring for now" 01187 raise Exception("The following alcas could not be found "+str(alcaList))
| def ConfigBuilder::ConfigBuilder::prepare_ALCAHARVEST | ( | self, | |
sequence = None |
|||
| ) |
Enrich the process with AlCaHarvesting step
Definition at line 1740 of file ConfigBuilder.py.
01741 : 01742 """ Enrich the process with AlCaHarvesting step """ 01743 harvestingConfig = self.loadAndRemember(self.ALCAHARVESTDefaultCFF) 01744 sequence=sequence.split(".")[-1] 01745 01746 # decide which AlcaHARVESTING paths to use 01747 harvestingList = sequence.split("+") 01748 for name in harvestingConfig.__dict__: 01749 harvestingstream = getattr(harvestingConfig,name) 01750 if name in harvestingList and isinstance(harvestingstream,cms.Path): 01751 self.schedule.append(harvestingstream) 01752 self.executeAndRemember("process.PoolDBOutputService.toPut.append(process.ALCAHARVEST" + name + "_dbOutput)") 01753 self.executeAndRemember("process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVEST" + name + "_metadata)") 01754 harvestingList.remove(name) 01755 # append the common part at the end of the sequence 01756 lastStep = getattr(harvestingConfig,"ALCAHARVESTDQMSaveAndMetadataWriter") 01757 self.schedule.append(lastStep) 01758 01759 if len(harvestingList) != 0 and 'dummyHarvesting' not in harvestingList : 01760 print "The following harvesting could not be found : ", harvestingList 01761 raise Exception("The following harvesting could not be found : "+str(harvestingList)) 01762 01763
| def ConfigBuilder::ConfigBuilder::prepare_ALCAOUTPUT | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1129 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_ALCAPRODUCER | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1126 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_CFWRITER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the crossing frame writer step
Definition at line 1308 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DATAMIX | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the digitisation step
Definition at line 1314 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DIGI | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the digitisation step
Definition at line 1295 of file ConfigBuilder.py.
01296 : 01297 """ Enrich the schedule with the digitisation step""" 01298 self.loadDefaultOrSpecifiedCFF(sequence,self.DIGIDefaultCFF) 01299 01300 if self._options.gflash==True: 01301 self.loadAndRemember("Configuration/StandardSequences/GFlashDIGI_cff") 01302 01303 if self._options.himix==True: 01304 self.loadAndRemember("SimGeneral/MixingModule/himixDIGI_cff") 01305 01306 self.scheduleSequence(sequence.split('.')[-1],'digitisation_step') 01307 return
| def ConfigBuilder::ConfigBuilder::prepare_DIGI2RAW | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1320 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DQM | ( | self, | |
sequence = 'DQMOffline' |
|||
| ) |
Definition at line 1674 of file ConfigBuilder.py.
01675 : 01676 # this one needs replacement 01677 01678 self.loadDefaultOrSpecifiedCFF(sequence,self.DQMOFFLINEDefaultCFF) 01679 sequenceList=sequence.split('.')[-1].split('+') 01680 from DQMOffline.Configuration.autoDQM import autoDQM 01681 self.expandMapping(sequenceList,autoDQM,index=0) 01682 01683 if len(set(sequenceList))!=len(sequenceList): 01684 sequenceList=list(set(sequenceList)) 01685 print "Duplicate entries for DQM:, using",sequenceList 01686 pathName='dqmoffline_step' 01687 01688 for (i,sequence) in enumerate(sequenceList): 01689 if (i!=0): 01690 pathName='dqmoffline_%d_step'%(i) 01691 01692 if 'HLT' in self.stepMap.keys() or self._options.hltProcess: 01693 self.renameHLTprocessInSequence(sequence) 01694 01695 # if both HLT and DQM are run in the same process, schedule [HLT]DQM in an EndPath 01696 if 'HLT' in self.stepMap.keys(): 01697 # need to put [HLT]DQM in an EndPath, to access the HLT trigger results 01698 #setattr(self.process,pathName, cms.EndPath( getattr(self.process, sequence ) ) ) 01699 self.scheduleSequence(sequence, pathName, what='EndPath') 01700 01701 else: 01702 # schedule DQM as a standard Path 01703 #setattr(self.process,pathName, cms.Path( getattr(self.process, sequence) ) ) 01704 self.scheduleSequence(sequence, pathName, what='Path') 01705 #self.schedule.append(getattr(self.process,pathName)) 01706
| def ConfigBuilder::ConfigBuilder::prepare_ENDJOB | ( | self, | |
sequence = 'endOfProcess' |
|||
| ) |
Definition at line 1764 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_FASTSIM | ( | self, | |
sequence = "all" |
|||
| ) |
Enrich the schedule with fastsim
Definition at line 1773 of file ConfigBuilder.py.
01774 : 01775 """Enrich the schedule with fastsim""" 01776 self.loadAndRemember("FastSimulation/Configuration/FamosSequences_cff") 01777 01778 if sequence in ('all','allWithHLTFiltering',''): 01779 if not 'HLT' in self.stepMap.keys(): 01780 self.prepare_HLT(sequence=None) 01781 01782 self.executeAndRemember("process.famosSimHits.SimulateCalorimetry = True") 01783 self.executeAndRemember("process.famosSimHits.SimulateTracking = True") 01784 01785 self.executeAndRemember("process.simulation = cms.Sequence(process.simulationWithFamos)") 01786 self.executeAndRemember("process.HLTEndSequence = cms.Sequence(process.reconstructionWithFamos)") 01787 01788 # since we have HLT here, the process should be called HLT 01789 self._options.name = "HLT" 01790 01791 # if we don't want to filter after HLT but simulate everything regardless of what HLT tells, we have to add reconstruction explicitly 01792 if sequence == 'all' and not 'HLT' in self.stepMap.keys(): #(a) 01793 self.finalizeFastSimHLT() 01794 elif sequence == 'famosWithEverything': 01795 self.process.fastsim_step = cms.Path( getattr(self.process, "famosWithEverything") ) 01796 self.schedule.append(self.process.fastsim_step) 01797 01798 # now the additional commands we need to make the config work 01799 self.executeAndRemember("process.VolumeBasedMagneticFieldESProducer.useParametrizedTrackerField = True") 01800 else: 01801 print "FastSim setting", sequence, "unknown." 01802 raise ValueError 01803 01804 if 'Flat' in self._options.beamspot: 01805 beamspotType = 'Flat' 01806 elif 'Gauss' in self._options.beamspot: 01807 beamspotType = 'Gaussian' 01808 else: 01809 beamspotType = 'BetaFunc' 01810 self.loadAndRemember('IOMC.EventVertexGenerators.VtxSmearedParameters_cfi') 01811 beamspotName = 'process.%sVtxSmearingParameters' %(self._options.beamspot) 01812 self.executeAndRemember(beamspotName+'.type = cms.string("%s")'%(beamspotType)) 01813 self.executeAndRemember('process.famosSimHits.VertexGenerator = '+beamspotName) 01814 if hasattr(self.process,'famosPileUp'): 01815 self.executeAndRemember('process.famosPileUp.VertexGenerator = '+beamspotName) 01816 01817
| def ConfigBuilder::ConfigBuilder::prepare_FILTER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with a user defined filter sequence
Definition at line 1433 of file ConfigBuilder.py.
01434 : 01435 ''' Enrich the schedule with a user defined filter sequence ''' 01436 ## load the relevant part 01437 filterConfig=self.load(sequence.split('.')[0]) 01438 filterSeq=sequence.split('.')[-1] 01439 ## print it in the configuration 01440 class PrintAllModules(object): 01441 def __init__(self): 01442 self.inliner='' 01443 pass 01444 def enter(self,visitee): 01445 try: 01446 label=visitee.label() 01447 ##needs to be in reverse order 01448 self.inliner=label+','+self.inliner 01449 except: 01450 pass 01451 def leave(self,v): pass 01452 01453 expander=PrintAllModules() 01454 getattr(self.process,filterSeq).visit( expander ) 01455 self._options.inlineObjets+=','+expander.inliner 01456 self._options.inlineObjets+=','+filterSeq 01457 01458 ## put the filtering path in the schedule 01459 self.scheduleSequence(filterSeq,'filtering_step') 01460 self.nextScheduleIsConditional=True 01461 ## put it before all the other paths 01462 self.productionFilterSequence = filterSeq 01463 01464 return
| def ConfigBuilder::ConfigBuilder::prepare_GEN | ( | self, | |
sequence = None |
|||
| ) |
load the fragment of generator configuration
Definition at line 1204 of file ConfigBuilder.py.
01205 : 01206 """ load the fragment of generator configuration """ 01207 loadFailure=False 01208 #remove trailing .py 01209 #support old style .cfi by changing into something.cfi into something_cfi 01210 #remove python/ from the name 01211 loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','') 01212 #standard location of fragments 01213 if not '/' in loadFragment: 01214 loadFragment='Configuration.Generator.'+loadFragment 01215 else: 01216 loadFragment=loadFragment.replace('/','.') 01217 try: 01218 print "Loading generator fragment from",loadFragment 01219 __import__(loadFragment) 01220 except: 01221 loadFailure=True 01222 #if self.process.source and self.process.source.type_()=='EmptySource': 01223 if not (self._options.filein or self._options.dbsquery): 01224 raise Exception("Neither gen fragment of input files provided: this is an inconsistent GEN step configuration") 01225 01226 if not loadFailure: 01227 generatorModule=sys.modules[loadFragment] 01228 genModules=generatorModule.__dict__ 01229 if self._options.hideGen: 01230 self.loadAndRemember(loadFragment) 01231 else: 01232 self.process.load(loadFragment) 01233 # expose the objects from that fragment to the configuration 01234 import FWCore.ParameterSet.Modules as cmstypes 01235 for name in genModules: 01236 theObject = getattr(generatorModule,name) 01237 if isinstance(theObject, cmstypes._Module): 01238 self._options.inlineObjets=name+','+self._options.inlineObjets 01239 elif isinstance(theObject, cms.Sequence) or isinstance(theObject, cmstypes.ESProducer): 01240 self._options.inlineObjets+=','+name 01241 01242 if sequence == self.GENDefaultSeq or sequence == 'pgen_genonly' or ( sequence == 'pgen_hi' and 'generator' in genModules): 01243 if 'ProductionFilterSequence' in genModules and ('generator' in genModules or 'hiSignal' in genModules): 01244 self.productionFilterSequence = 'ProductionFilterSequence' 01245 elif 'generator' in genModules: 01246 self.productionFilterSequence = 'generator' 01247 01248 """ Enrich the schedule with the rest of the generation step """ 01249 self.loadDefaultOrSpecifiedCFF(sequence,self.GENDefaultCFF) 01250 genSeqName=sequence.split('.')[-1] 01251 01252 if not 'FASTSIM' in self.stepMap: 01253 try: 01254 from Configuration.StandardSequences.VtxSmeared import VtxSmeared 01255 self.loadAndRemember(VtxSmeared[self._options.beamspot]) 01256 except ImportError: 01257 raise Exception("VertexSmearing type or beamspot "+self._options.beamspot+" unknown.") 01258 01259 if self._options.scenario == 'HeavyIons' and self._options.himix: 01260 self.loadAndRemember("SimGeneral/MixingModule/himixGEN_cff") 01261 01262 self.process.generation_step = cms.Path( getattr(self.process,genSeqName) ) 01263 self.schedule.append(self.process.generation_step) 01264 01265 #register to the genstepfilter the name of the path (static right now, but might evolve) 01266 self.executeAndRemember('process.genstepfilter.triggerConditions=cms.vstring("generation_step")') 01267 01268 if 'reGEN' in self.stepMap: 01269 #stop here 01270 return 01271 01272 """ Enrich the schedule with the summary of the filter step """ 01273 #the gen filter in the endpath 01274 self.loadAndRemember("GeneratorInterface/Core/genFilterSummary_cff") 01275 self.scheduleSequenceAtEnd('genFilterSummary','genfiltersummary_step') 01276 return
| def ConfigBuilder::ConfigBuilder::prepare_HARVESTING | ( | self, | |
sequence = None |
|||
| ) |
Enrich the process with harvesting step
Definition at line 1707 of file ConfigBuilder.py.
01708 : 01709 """ Enrich the process with harvesting step """ 01710 self.EDMtoMECFF='Configuration/StandardSequences/EDMtoME'+self._options.harvesting+'_cff' 01711 self.loadAndRemember(self.EDMtoMECFF) 01712 self.scheduleSequence('EDMtoME','edmtome_step') 01713 01714 harvestingConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.HARVESTINGDefaultCFF) 01715 sequence = sequence.split('.')[-1] 01716 01717 # decide which HARVESTING paths to use 01718 harvestingList = sequence.split("+") 01719 from DQMOffline.Configuration.autoDQM import autoDQM 01720 self.expandMapping(harvestingList,autoDQM,index=1) 01721 01722 if len(set(harvestingList))!=len(harvestingList): 01723 harvestingList=list(set(harvestingList)) 01724 print "Duplicate entries for HARVESTING, using",harvestingList 01725 01726 for name in harvestingList: 01727 if not name in harvestingConfig.__dict__: 01728 print name,"is not a possible harvesting type. Available are",harvestingConfig.__dict__.keys() 01729 continue 01730 harvestingstream = getattr(harvestingConfig,name) 01731 if isinstance(harvestingstream,cms.Path): 01732 self.schedule.append(harvestingstream) 01733 self.blacklist_paths.append(harvestingstream) 01734 if isinstance(harvestingstream,cms.Sequence): 01735 setattr(self.process,name+"_step",cms.Path(harvestingstream)) 01736 self.schedule.append(getattr(self.process,name+"_step")) 01737 01738 self.scheduleSequence('DQMSaver','dqmsave_step') 01739 return
| def ConfigBuilder::ConfigBuilder::prepare_HLT | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the HLT simulation step
Definition at line 1355 of file ConfigBuilder.py.
01356 : 01357 """ Enrich the schedule with the HLT simulation step""" 01358 if not sequence: 01359 print "no specification of the hlt menu has been given, should never happen" 01360 raise Exception('no HLT sequence provided') 01361 01362 if '@' in sequence: 01363 # case where HLT:@something was provided 01364 from Configuration.HLT.autoHLT import autoHLT 01365 key = sequence[1:] 01366 if key in autoHLT: 01367 sequence = autoHLT[key] 01368 else: 01369 raise ValueError('no HLT mapping key "%s" found in autoHLT' % key) 01370 01371 if ',' in sequence: 01372 # case where HLT:something:something was provided 01373 self.executeAndRemember('import HLTrigger.Configuration.Utilities') 01374 optionsForHLT = {} 01375 if self._options.scenario == 'HeavyIons': 01376 optionsForHLT['type'] = 'HIon' 01377 else: 01378 optionsForHLT['type'] = 'GRun' 01379 optionsForHLTConfig = ', '.join('%s=%s' % (key, repr(val)) for (key, val) in optionsForHLT.iteritems()) 01380 self.executeAndRemember('process.loadHltConfiguration("%s",%s)'%(sequence.replace(',',':'),optionsForHLTConfig)) 01381 else: 01382 if 'FASTSIM' in self.stepMap: 01383 self.loadAndRemember('HLTrigger/Configuration/HLT_%s_Famos_cff' % sequence) 01384 else: 01385 self.loadAndRemember('HLTrigger/Configuration/HLT_%s_cff' % sequence) 01386 01387 if self._options.isMC: 01388 self._options.customisation_file+=",HLTrigger/Configuration/customizeHLTforMC.customizeHLTforMC" 01389 01390 if self._options.name != 'HLT': 01391 self.additionalCommands.append('from HLTrigger.Configuration.CustomConfigs import ProcessName') 01392 self.additionalCommands.append('process = ProcessName(process)') 01393 self.additionalCommands.append('') 01394 from HLTrigger.Configuration.CustomConfigs import ProcessName 01395 self.process = ProcessName(self.process) 01396 01397 self.schedule.append(self.process.HLTSchedule) 01398 [self.blacklist_paths.append(path) for path in self.process.HLTSchedule if isinstance(path,(cms.Path,cms.EndPath))] 01399 if ('FASTSIM' in self.stepMap and 'HLT' in self.stepMap): 01400 self.finalizeFastSimHLT() 01401
| def ConfigBuilder::ConfigBuilder::prepare_L1 | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the L1 simulation step
Definition at line 1330 of file ConfigBuilder.py.
01331 : 01332 """ Enrich the schedule with the L1 simulation step""" 01333 if not sequence: 01334 self.loadAndRemember(self.L1EMDefaultCFF) 01335 else: 01336 # let the L1 package decide for the scenarios available 01337 from L1Trigger.Configuration.ConfigBuilder import getConfigsForScenario 01338 listOfImports = getConfigsForScenario(sequence) 01339 for file in listOfImports: 01340 self.loadAndRemember(file) 01341 self.scheduleSequence('SimL1Emulator','L1simulation_step') 01342 return
| def ConfigBuilder::ConfigBuilder::prepare_L1HwVal | ( | self, | |
sequence = 'L1HwVal' |
|||
| ) |
Enrich the schedule with L1 HW validation
Definition at line 1420 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_L1Reco | ( | self, | |
sequence = "L1Reco" |
|||
| ) |
Enrich the schedule with L1 reconstruction
Definition at line 1427 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_L1REPACK | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection
Definition at line 1343 of file ConfigBuilder.py.
01344 : 01345 """ Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection""" 01346 if sequence is not 'GT': 01347 print 'Running the full L1 emulator is not supported yet' 01348 raise Exception('unsupported feature') 01349 if sequence is 'GT': 01350 self.loadAndRemember('Configuration/StandardSequences/SimL1EmulatorRepack_GT_cff') 01351 if self._options.scenario == 'HeavyIons': 01352 self.renameInputTagsInSequence("SimL1Emulator","rawDataCollector","rawDataRepacker") 01353 self.scheduleSequence('SimL1Emulator','L1simulation_step') 01354
| def ConfigBuilder::ConfigBuilder::prepare_LHE | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1188 of file ConfigBuilder.py.
01189 : 01190 #load the fragment 01191 ##make it loadable 01192 loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','').replace('/','.') 01193 print "Loading lhe fragment from",loadFragment 01194 __import__(loadFragment) 01195 self.process.load(loadFragment) 01196 ##inline the modules 01197 self._options.inlineObjets+=','+sequence 01198 01199 getattr(self.process,sequence).nEvents = int(self._options.number) 01200 01201 #schedule it 01202 self.process.lhe_step = cms.Path( getattr( self.process,sequence) ) 01203 self.schedule.append( self.process.lhe_step )
| def ConfigBuilder::ConfigBuilder::prepare_POSTRECO | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the postreco step
Definition at line 1521 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RAW2DIGI | ( | self, | |
sequence = "RawToDigi" |
|||
| ) |
Definition at line 1413 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RAW2RECO | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1402 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RECO | ( | self, | |
sequence = "reconstruction" |
|||
| ) |
Enrich the schedule with reconstruction
Definition at line 1465 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_REPACK | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1325 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_SIM | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the simulation step
Definition at line 1277 of file ConfigBuilder.py.
01278 : 01279 """ Enrich the schedule with the simulation step""" 01280 self.loadAndRemember(self.SIMDefaultCFF) 01281 if self._options.gflash==True: 01282 self.loadAndRemember("Configuration/StandardSequences/GFlashSIM_cff") 01283 01284 if self._options.magField=='0T': 01285 self.executeAndRemember("process.g4SimHits.UseMagneticField = cms.bool(False)") 01286 01287 if self._options.himix==True: 01288 if self._options.geometry in defaultOptions.geometryExtendedOptions: 01289 self.loadAndRemember("SimGeneral/MixingModule/himixSIMExtended_cff") 01290 else: 01291 self.loadAndRemember("SimGeneral/MixingModule/himixSIMIdeal_cff") 01292 01293 self.scheduleSequence('psim','simulation_step') 01294 return
| def ConfigBuilder::ConfigBuilder::prepare_SKIM | ( | self, | |
sequence = "all" |
|||
| ) |
Enrich the schedule with skimming fragments
Definition at line 1471 of file ConfigBuilder.py.
01472 : 01473 ''' Enrich the schedule with skimming fragments''' 01474 skimConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.SKIMDefaultCFF) 01475 sequence = sequence.split('.')[-1] 01476 01477 skimlist=sequence.split('+') 01478 ## support @Mu+DiJet+@Electron configuration via autoSkim.py 01479 from Configuration.Skimming.autoSkim import autoSkim 01480 self.expandMapping(skimlist,autoSkim) 01481 01482 #print "dictionnary for skims:",skimConfig.__dict__ 01483 for skim in skimConfig.__dict__: 01484 skimstream = getattr(skimConfig,skim) 01485 if isinstance(skimstream,cms.Path): 01486 #black list the alca path so that they do not appear in the cfg 01487 self.blacklist_paths.append(skimstream) 01488 if (not isinstance(skimstream,cms.FilteredStream)): 01489 continue 01490 shortname = skim.replace('SKIMStream','') 01491 if (sequence=="all"): 01492 self.addExtraStream(skim,skimstream) 01493 elif (shortname in skimlist): 01494 self.addExtraStream(skim,skimstream) 01495 #add a DQM eventcontent for this guy 01496 if self._options.datatier!="": 01497 self.process.load(self.EVTCONTDefaultCFF) 01498 skimstreamDQM = cms.FilteredStream( 01499 responsible = skimstream.responsible, 01500 name = skimstream.name+'DQM', 01501 paths = skimstream.paths, 01502 selectEvents = skimstream.selectEvents, 01503 content = self._options.datatier+'EventContent', 01504 dataTier = cms.untracked.string(self._options.datatier) 01505 ) 01506 self.addExtraStream(skim+'DQM',skimstreamDQM) 01507 for i in range(skimlist.count(shortname)): 01508 skimlist.remove(shortname) 01509 01510 01511 01512 if (skimlist.__len__()!=0 and sequence!="all"): 01513 print 'WARNING, possible typo with SKIM:'+'+'.join(skimlist) 01514 raise Exception('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
| def ConfigBuilder::ConfigBuilder::prepare_USER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with a user defined sequence
Definition at line 1515 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_VALIDATION | ( | self, | |
sequence = 'validation' |
|||
| ) |
Definition at line 1528 of file ConfigBuilder.py.
01529 : 01530 self.loadDefaultOrSpecifiedCFF(sequence,self.VALIDATIONDefaultCFF) 01531 #in case VALIDATION:something:somethingelse -> something,somethingelse 01532 sequence=sequence.split('.')[-1] 01533 if sequence.find(',')!=-1: 01534 prevalSeqName=sequence.split(',')[0] 01535 valSeqName=sequence.split(',')[1] 01536 else: 01537 postfix='' 01538 if sequence: 01539 postfix='_'+sequence 01540 prevalSeqName='prevalidation'+postfix 01541 valSeqName='validation'+postfix 01542 if not hasattr(self.process,valSeqName): 01543 prevalSeqName='' 01544 valSeqName=sequence 01545 01546 if not 'DIGI' in self.stepMap and not 'FASTSIM' in self.stepMap and not valSeqName.startswith('genvalid'): 01547 if self._options.restoreRNDSeeds==False and not self._options.restoreRNDSeeds==True: 01548 self._options.restoreRNDSeeds=True 01549 01550 #rename the HLT process in validation steps 01551 if ('HLT' in self.stepMap and not 'FASTSIM' in self.stepMap) or self._options.hltProcess: 01552 self.renameHLTprocessInSequence(valSeqName) 01553 if prevalSeqName: 01554 self.renameHLTprocessInSequence(prevalSeqName) 01555 01556 if prevalSeqName: 01557 self.process.prevalidation_step = cms.Path( getattr(self.process, prevalSeqName ) ) 01558 self.schedule.append(self.process.prevalidation_step) 01559 01560 self.process.validation_step = cms.EndPath( getattr(self.process,valSeqName ) ) 01561 self.schedule.append(self.process.validation_step) 01562 01563 if not 'DIGI' in self.stepMap and not 'FASTSIM' in self.stepMap: 01564 self.executeAndRemember("process.mix.playback = True") 01565 01566 if hasattr(self.process,"genstepfilter") and len(self.process.genstepfilter.triggerConditions): 01567 #will get in the schedule, smoothly 01568 self.process.validation_step._seq = self.process.genstepfilter * self.process.validation_step._seq 01569 01570 return 01571
| def ConfigBuilder::ConfigBuilder::profileOptions | ( | self | ) |
addIgProfService Function to add the igprof profile service so that you can dump in the middle of the run.
Definition at line 214 of file ConfigBuilder.py.
00215 : 00216 """ 00217 addIgProfService 00218 Function to add the igprof profile service so that you can dump in the middle 00219 of the run. 00220 """ 00221 profileOpts = self._options.profile.split(':') 00222 profilerStart = 1 00223 profilerInterval = 100 00224 profilerFormat = None 00225 profilerJobFormat = None 00226 00227 if len(profileOpts): 00228 #type, given as first argument is unused here 00229 profileOpts.pop(0) 00230 if len(profileOpts): 00231 startEvent = profileOpts.pop(0) 00232 if not startEvent.isdigit(): 00233 raise Exception("%s is not a number" % startEvent) 00234 profilerStart = int(startEvent) 00235 if len(profileOpts): 00236 eventInterval = profileOpts.pop(0) 00237 if not eventInterval.isdigit(): 00238 raise Exception("%s is not a number" % eventInterval) 00239 profilerInterval = int(eventInterval) 00240 if len(profileOpts): 00241 profilerFormat = profileOpts.pop(0) 00242 00243 00244 if not profilerFormat: 00245 profilerFormat = "%s___%s___%s___%s___%s___%s___%%I.gz" % (self._options.evt_type.replace("_cfi", ""), 00246 self._options.step, 00247 self._options.pileup, 00248 self._options.conditions, 00249 self._options.datatier, 00250 self._options.profileTypeLabel) 00251 if not profilerJobFormat and profilerFormat.endswith(".gz"): 00252 profilerJobFormat = profilerFormat.replace(".gz", "_EndOfJob.gz") 00253 elif not profilerJobFormat: 00254 profilerJobFormat = profilerFormat + "_EndOfJob.gz" 00255 00256 return (profilerStart,profilerInterval,profilerFormat,profilerJobFormat)
| def ConfigBuilder::ConfigBuilder::renameHLTprocessInSequence | ( | self, | |
| sequence, | |||
proc = None, |
|||
HLTprocess = 'HLT' |
|||
| ) |
Definition at line 1641 of file ConfigBuilder.py.
01642 : 01643 if self._options.hltProcess: 01644 proc=self._options.hltProcess 01645 else: 01646 proc=self.process.name_() 01647 if proc==HLTprocess: return 01648 # look up all module in dqm sequence 01649 print "replacing %s process name - sequence %s will use '%s'" % (HLTprocess,sequence, proc) 01650 getattr(self.process,sequence).visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor(HLTprocess,proc,whitelist = ("subSystemFolder",))) 01651 if 'from Configuration.PyReleaseValidation.ConfigBuilder import ConfigBuilder' not in self.additionalCommands: 01652 self.additionalCommands.append('from Configuration.PyReleaseValidation.ConfigBuilder import ConfigBuilder') 01653 self.additionalCommands.append('process.%s.visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor("%s", "%s", whitelist = ("subSystemFolder",)))'% (sequence,HLTprocess, proc)) 01654
| def ConfigBuilder::ConfigBuilder::renameInputTagsInSequence | ( | self, | |
| sequence, | |||
oldT = "rawDataCollector", |
|||
newT = "rawDataRepacker" |
|||
| ) |
Definition at line 1631 of file ConfigBuilder.py.
01632 : 01633 print "Replacing all InputTag %s => %s"%(oldT,newT) 01634 from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag 01635 massSearchReplaceAnyInputTag(getattr(self.process,sequence),oldT,newT) 01636 loadMe='from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag' 01637 if not loadMe in self.additionalCommands: 01638 self.additionalCommands.append(loadMe) 01639 self.additionalCommands.append('massSearchReplaceAnyInputTag(process.%s,"%s","%s",False)'%(sequence,oldT,newT))
| def ConfigBuilder::ConfigBuilder::scheduleSequence | ( | self, | |
| seq, | |||
| prefix, | |||
what = 'Path' |
|||
| ) |
Definition at line 1095 of file ConfigBuilder.py.
01096 : 01097 if not what in ['Path','EndPath']: 01098 raise Exception("Can only schedule a Path or and EndPath."+str(what)+"is not allowed") 01099 if '*' in seq: 01100 #create only one path with all sequences in it 01101 for i,s in enumerate(seq.split('*')): 01102 if i==0: 01103 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, s) )) 01104 else: 01105 p=getattr(self.process,prefix) 01106 p+=getattr(self.process, s) 01107 self.schedule.append(getattr(self.process,prefix)) 01108 return 01109 else: 01110 #create as many path as many sequences 01111 if not '+' in seq: 01112 if self.nextScheduleIsConditional and what=='Path': 01113 self.conditionalPaths.append(prefix) 01114 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, seq) )) 01115 self.schedule.append(getattr(self.process,prefix)) 01116 else: 01117 for i,s in enumerate(seq.split('+')): 01118 sn=prefix+'%d'%(i) 01119 setattr(self.process,sn,getattr(cms,what)( getattr(self.process, s) )) 01120 self.schedule.append(getattr(self.process,sn)) 01121 return
| def ConfigBuilder::ConfigBuilder::scheduleSequenceAtEnd | ( | self, | |
| seq, | |||
| prefix | |||
| ) |
Definition at line 1122 of file ConfigBuilder.py.
ConfigBuilder::ConfigBuilder::_options [private] |
Definition at line 154 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 1707 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
load the relevant part
needs to be in reverse order
print it in the configuration
Definition at line 1435 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
put the filtering path in the schedule
Definition at line 154 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
put it before all the other paths
Definition at line 154 of file ConfigBuilder.py.
Definition at line 1829 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 794 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
Definition at line 154 of file ConfigBuilder.py.
1.7.3