|
|
|
The main building routines
Definition at line 150 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::__init__ | ( | self, | |
| options, | |||
process = None, |
|||
with_output = False, |
|||
with_input = False |
|||
| ) |
options taken from old cmsDriver and optparse
Definition at line 153 of file ConfigBuilder.py.
00154 : 00155 """options taken from old cmsDriver and optparse """ 00156 00157 options.outfile_name = options.dirout+options.fileout 00158 00159 self._options = options 00160 00161 if self._options.isData and options.isMC: 00162 raise Exception("ERROR: You may specify only --data or --mc, not both") 00163 #if not self._options.conditions: 00164 # raise Exception("ERROR: No conditions given!\nPlease specify conditions. E.g. via --conditions=IDEAL_30X::All") 00165 00166 if hasattr(self._options,"datatier") and self._options.datatier and 'DQMROOT' in self._options.datatier and 'ENDJOB' in self._options.step: 00167 self._options.step=self._options.step.replace(',ENDJOB','') 00168 00169 # what steps are provided by this class? 00170 stepList = [re.sub(r'^prepare_', '', methodName) for methodName in ConfigBuilder.__dict__ if methodName.startswith('prepare_')] 00171 self.stepMap={} 00172 for step in self._options.step.split(","): 00173 if step=='': continue 00174 stepParts = step.split(":") 00175 stepName = stepParts[0] 00176 if stepName not in stepList and not stepName.startswith('re'): 00177 raise ValueError("Step "+stepName+" unknown") 00178 if len(stepParts)==1: 00179 self.stepMap[stepName]="" 00180 elif len(stepParts)==2: 00181 self.stepMap[stepName]=stepParts[1].split('+') 00182 elif len(stepParts)==3: 00183 self.stepMap[stepName]=(stepParts[2].split('+'),stepParts[1]) 00184 else: 00185 raise ValueError("Step definition "+step+" invalid") 00186 #print "map of steps is:",self.stepMap 00187 00188 self.with_output = with_output 00189 if hasattr(self._options,"no_output_flag") and self._options.no_output_flag: 00190 self.with_output = False 00191 self.with_input = with_input 00192 if process == None: 00193 self.process = cms.Process(self._options.name) 00194 else: 00195 self.process = process 00196 self.imports = [] 00197 self.define_Configs() 00198 self.schedule = list() 00199 00200 # we are doing three things here: 00201 # creating a process to catch errors 00202 # building the code to re-create the process 00203 00204 self.additionalCommands = [] 00205 # TODO: maybe a list of to be dumped objects would help as well 00206 self.blacklist_paths = [] 00207 self.addedObjects = [] 00208 self.additionalOutputs = {} 00209 00210 self.productionFilterSequence = None
| def ConfigBuilder::ConfigBuilder::addCommon | ( | self | ) |
Definition at line 270 of file ConfigBuilder.py.
00271 : 00272 if 'HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys(): 00273 self.process.options = cms.untracked.PSet( Rethrow = cms.untracked.vstring('ProductNotFound'),fileMode = cms.untracked.string('FULLMERGE')) 00274 else: 00275 self.process.options = cms.untracked.PSet( ) 00276 self.addedObjects.append(("","options")) 00277 00278 if self._options.lazy_download: 00279 self.process.AdaptorConfig = cms.Service("AdaptorConfig", 00280 stats = cms.untracked.bool(True), 00281 enable = cms.untracked.bool(True), 00282 cacheHint = cms.untracked.string("lazy-download"), 00283 readHint = cms.untracked.string("read-ahead-buffered") 00284 ) 00285 self.addedObjects.append(("Setup lazy download","AdaptorConfig")) 00286 00287 #self.process.cmsDriverCommand = cms.untracked.PSet( command=cms.untracked.string('cmsDriver.py '+self._options.arguments) ) 00288 #self.addedObjects.append(("what cmsDriver command was used","cmsDriverCommand")) 00289 00290 if self._options.profile: 00291 (start, interval, eventFormat, jobFormat)=self.profileOptions() 00292 self.process.IgProfService = cms.Service("IgProfService", 00293 reportFirstEvent = cms.untracked.int32(start), 00294 reportEventInterval = cms.untracked.int32(interval), 00295 reportToFileAtPostEvent = cms.untracked.string("| gzip -c > %s"%(eventFormat)), 00296 reportToFileAtPostEndJob = cms.untracked.string("| gzip -c > %s"%(jobFormat))) 00297 self.addedObjects.append(("Setup IGProf Service for profiling","IgProfService"))
| def ConfigBuilder::ConfigBuilder::addConditions | ( | self | ) |
Add conditions to the process
Definition at line 661 of file ConfigBuilder.py.
00662 : 00663 """Add conditions to the process""" 00664 if not self._options.conditions: return 00665 00666 if 'auto:' in self._options.conditions: 00667 from Configuration.AlCa.autoCond import autoCond 00668 key=self._options.conditions.split(':')[-1] 00669 if key not in autoCond: 00670 raise Exception('no correspondance for '+self._options.conditions+'\n available keys are'+','.join(autoCond.keys())) 00671 else: 00672 self._options.conditions = autoCond[key] 00673 00674 # the option can be a list of GT name and connection string 00675 00676 if isinstance(self._options.conditions,tuple): 00677 if self._options.custom_conditions: 00678 self._options.custom_conditions+='+'+self._options.conditions[1] 00679 else: 00680 self._options.custom_conditions=self._options.conditions[1] 00681 self._options.conditions=self._options.conditions[0] 00682 00683 00684 if 'FrontierConditions_GlobalTag' in self._options.conditions: 00685 print 'using FrontierConditions_GlobalTag in --conditions is not necessary anymore and will be deprecated soon. please update your command line' 00686 self._options.conditions = self._options.conditions.replace("FrontierConditions_GlobalTag,",'') 00687 00688 conditions = self._options.conditions.split(',') 00689 00690 gtName = str( conditions[0] ) 00691 if len(conditions) > 1: 00692 connect = str( conditions[1] ) 00693 if len(conditions) > 2: 00694 pfnPrefix = str( conditions[2] ) 00695 00696 self.loadAndRemember(self.ConditionsDefaultCFF) 00697 00698 # set the global tag 00699 self.executeAndRemember("process.GlobalTag.globaltag = '%s'" % gtName) 00700 if len(conditions) > 1: 00701 self.executeAndRemember("process.GlobalTag.connect = '%s'" % connect) 00702 if len(conditions) > 2: 00703 self.executeAndRemember("process.GlobalTag.pfnPrefix = cms.untracked.string('%s')" % pfnPrefix) 00704 00705 if self._options.slhc: 00706 self.loadAndRemember("SLHCUpgradeSimulations/Geometry/fakeConditions_%s_cff"%(self._options.slhc,)) 00707 00708 if self._options.custom_conditions!="": 00709 specs=self._options.custom_conditions.split('+') 00710 self.executeAndRemember("process.GlobalTag.toGet = cms.VPSet()") 00711 for spec in specs: 00712 #format is tag=<...>,record=<...>,connect=<...>,label=<...> with connect and label optionnal 00713 items=spec.split(',') 00714 payloadSpec={} 00715 allowedFields=['tag','record','connect','label'] 00716 for i,item in enumerate(items): 00717 if '=' in item: 00718 field=item.split('=')[0] 00719 if not field in allowedFields: 00720 raise Exception('in --custom_conditions, '+field+' is not a valid field') 00721 payloadSpec[field]=item.split('=')[1] 00722 else: 00723 payloadSpec[allowedFields[i]]=item 00724 if (not 'record' in payloadSpec) or (not 'tag' in payloadSpec): 00725 raise Exception('conditions cannot be customised with: '+repr(payloadSpec)+' no record or tag field available') 00726 payloadSpecToAppend='' 00727 for i,item in enumerate(allowedFields): 00728 if not item in payloadSpec: continue 00729 if not payloadSpec[item]: continue 00730 if i<2: untracked='' 00731 else: untracked='untracked.' 00732 payloadSpecToAppend+='%s=cms.%sstring("%s"),'%(item,untracked,payloadSpec[item]) 00733 print 'customising the GlogalTag with:',payloadSpecToAppend 00734 self.executeAndRemember('process.GlobalTag.toGet.append(cms.PSet(%s))'%(payloadSpecToAppend,))
| def ConfigBuilder::ConfigBuilder::addCustomise | ( | self | ) |
Include the customise code
Definition at line 735 of file ConfigBuilder.py.
00736 : 00737 """Include the customise code """ 00738 00739 custOpt=self._options.customisation_file.split(",") 00740 custMap={} 00741 for opt in custOpt: 00742 if opt=='': continue 00743 if opt.count('.')>1: 00744 raise Exception("more than . in the specification:"+opt) 00745 fileName=opt.split('.')[0] 00746 if opt.count('.')==0: rest='customise' 00747 else: 00748 rest=opt.split('.')[1] 00749 if rest=='py': rest='customise' #catch the case of --customise file.py 00750 00751 if fileName in custMap: 00752 custMap[fileName].extend(rest.split('+')) 00753 else: 00754 custMap[fileName]=rest.split('+') 00755 00756 if len(custMap)==0: 00757 final_snippet='\n' 00758 else: 00759 final_snippet='\n# customisation of the process.\n' 00760 00761 allFcn=[] 00762 for opt in custMap: 00763 allFcn.extend(custMap[opt]) 00764 for fcn in allFcn: 00765 if allFcn.count(fcn)!=1: 00766 raise Exception("cannot specify twice "+fcn+" as a customisation method") 00767 00768 for f in custMap: 00769 # let python search for that package and do syntax checking at the same time 00770 packageName = f.replace(".py","").replace("/",".") 00771 __import__(packageName) 00772 package = sys.modules[packageName] 00773 00774 # now ask the package for its definition and pick .py instead of .pyc 00775 customiseFile = re.sub(r'\.pyc$', '.py', package.__file__) 00776 00777 final_snippet+='\n# Automatic addition of the customisation function from '+packageName+'\n' 00778 if self._options.inline_custom: 00779 for line in file(customiseFile,'r'): 00780 if "import FWCore.ParameterSet.Config" in line: 00781 continue 00782 final_snippet += line 00783 else: 00784 final_snippet += 'from %s import %s \n'%(packageName,','.join(custMap[f])) 00785 for fcn in custMap[f]: 00786 print "customising the process with",fcn,"from",f 00787 if not hasattr(package,fcn): 00788 #bound to fail at run time 00789 raise Exception("config "+f+" has no function "+fcn) 00790 #execute the command 00791 self.process=getattr(package,fcn)(self.process) 00792 #and print it in the configuration 00793 final_snippet += "\n#call to customisation function "+fcn+" imported from "+packageName 00794 final_snippet += "\nprocess = %s(process)\n"%(fcn,) 00795 00796 if len(custMap)!=0: 00797 final_snippet += '\n# End of customisation functions\n' 00798 00799 ### now for a usuful command 00800 if self._options.customise_commands: 00801 import string 00802 final_snippet +='\n# Customisation from command line' 00803 for com in self._options.customise_commands.split('\\n'): 00804 com=string.lstrip(com) 00805 self.executeAndRemember(com) 00806 final_snippet +='\n'+com 00807 00808 return final_snippet
| def ConfigBuilder::ConfigBuilder::addExtraStream | ( | self, | |
| name, | |||
| stream, | |||
workflow = 'full' |
|||
| ) |
Definition at line 1034 of file ConfigBuilder.py.
01035 : 01036 # define output module and go from there 01037 output = cms.OutputModule("PoolOutputModule") 01038 if stream.selectEvents.parameters_().__len__()!=0: 01039 output.SelectEvents = stream.selectEvents 01040 else: 01041 output.SelectEvents = cms.untracked.PSet() 01042 output.SelectEvents.SelectEvents=cms.vstring() 01043 if isinstance(stream.paths,tuple): 01044 for path in stream.paths: 01045 output.SelectEvents.SelectEvents.append(path.label()) 01046 else: 01047 output.SelectEvents.SelectEvents.append(stream.paths.label()) 01048 01049 01050 01051 if isinstance(stream.content,str): 01052 evtPset=getattr(self.process,stream.content) 01053 for p in evtPset.parameters_(): 01054 setattr(output,p,getattr(evtPset,p)) 01055 if not self._options.inlineEventContent: 01056 def doNotInlineEventContent(instance,label = "process."+stream.content+".outputCommands"): 01057 return label 01058 output.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 01059 else: 01060 output.outputCommands = stream.content 01061 01062 01063 output.fileName = cms.untracked.string(self._options.dirout+stream.name+'.root') 01064 01065 output.dataset = cms.untracked.PSet( dataTier = stream.dataTier, 01066 filterName = cms.untracked.string(stream.name)) 01067 01068 if self._options.filtername: 01069 output.dataset.filterName= cms.untracked.string(self._options.filtername+"_"+stream.name) 01070 01071 #add an automatic flushing to limit memory consumption 01072 output.eventAutoFlushCompressedSize=cms.untracked.int32(5*1024*1024) 01073 01074 if workflow in ("producers,full"): 01075 if isinstance(stream.paths,tuple): 01076 for path in stream.paths: 01077 self.schedule.append(path) 01078 else: 01079 self.schedule.append(stream.paths) 01080 01081 01082 # in case of relvals we don't want to have additional outputs 01083 if (not self._options.relval) and workflow in ("full","output"): 01084 self.additionalOutputs[name] = output 01085 setattr(self.process,name,output) 01086 01087 if workflow == 'output': 01088 # adjust the select events to the proper trigger results from previous process 01089 filterList = output.SelectEvents.SelectEvents 01090 for i, filter in enumerate(filterList): 01091 filterList[i] = filter+":"+self._options.triggerResultsProcess 01092 01093 return output
| def ConfigBuilder::ConfigBuilder::addMaxEvents | ( | self | ) |
Here we decide how many evts will be processed
Definition at line 298 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::addOutput | ( | self | ) |
Add output module to the process
Definition at line 416 of file ConfigBuilder.py.
00417 : 00418 """ Add output module to the process """ 00419 result="" 00420 if self._options.outputDefinition: 00421 if self._options.datatier: 00422 print "--datatier & --eventcontent options ignored" 00423 00424 def anyOf(listOfKeys,dict,opt=None): 00425 for k in listOfKeys: 00426 if k in dict: 00427 toReturn=dict[k] 00428 dict.pop(k) 00429 return toReturn 00430 if opt!=None: 00431 return opt 00432 else: 00433 raise Exception("any of "+','.join(listOfKeys)+" are mandatory entries of --output options") 00434 00435 #new output convention with a list of dict 00436 outList = eval(self._options.outputDefinition) 00437 for (id,outDefDict) in enumerate(outList): 00438 outDefDictStr=outDefDict.__str__() 00439 if not isinstance(outDefDict,dict): 00440 raise Exception("--output needs to be passed a list of dict"+self._options.outputDefinition+" is invalid") 00441 #requires option: tier 00442 theTier=anyOf(['t','tier','dataTier'],outDefDict) 00443 #optional option: eventcontent, filtername, selectEvents, moduleLabel, filename 00444 ## event content 00445 theStreamType=anyOf(['e','ec','eventContent','streamType'],outDefDict,theTier) 00446 theFilterName=anyOf(['f','ftN','filterName'],outDefDict,'') 00447 theSelectEvent=anyOf(['s','sE','selectEvents'],outDefDict,'') 00448 theModuleLabel=anyOf(['l','mL','moduleLabel'],outDefDict,'') 00449 theExtraOutputCommands=anyOf(['o','oC','outputCommands'],outDefDict,'') 00450 # module label has a particular role 00451 if not theModuleLabel: 00452 tryNames=[theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+'output', 00453 theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+'output', 00454 theStreamType.replace(theTier.replace('-',''),'')+theTier.replace('-','')+theFilterName+theSelectEvent.split(',')[0].replace(':','for').replace(' ','')+'output' 00455 ] 00456 for name in tryNames: 00457 if not hasattr(self.process,name): 00458 theModuleLabel=name 00459 break 00460 if not theModuleLabel: 00461 raise Exception("cannot find a module label for specification: "+outDefDictStr) 00462 if id==0: 00463 defaultFileName=self._options.outfile_name 00464 else: 00465 defaultFileName=self._options.outfile_name.replace('.root','_in'+theTier+'.root') 00466 00467 theFileName=self._options.dirout+anyOf(['fn','fileName'],outDefDict,defaultFileName) 00468 if not theFileName.endswith('.root'): 00469 theFileName+='.root' 00470 00471 if len(outDefDict.keys()): 00472 raise Exception("unused keys from --output options: "+','.join(outDefDict.keys())) 00473 if theStreamType=='DQMROOT': theStreamType='DQM' 00474 if theStreamType=='ALL': 00475 theEventContent = cms.PSet(outputCommands = cms.untracked.vstring('keep *')) 00476 else: 00477 theEventContent = getattr(self.process, theStreamType+"EventContent") 00478 00479 if theStreamType=='ALCARECO' and not theFilterName: 00480 theFilterName='StreamALCACombined' 00481 00482 CppType='PoolOutputModule' 00483 if theStreamType=='DQM' and theTier=='DQMROOT': CppType='DQMRootOutputModule' 00484 output = cms.OutputModule(CppType, 00485 theEventContent.clone(), 00486 fileName = cms.untracked.string(theFileName), 00487 dataset = cms.untracked.PSet( 00488 dataTier = cms.untracked.string(theTier), 00489 filterName = cms.untracked.string(theFilterName)) 00490 ) 00491 if not theSelectEvent and hasattr(self.process,'generation_step'): 00492 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step')) 00493 if theSelectEvent: 00494 output.SelectEvents =cms.untracked.PSet(SelectEvents = cms.vstring(theSelectEvent)) 00495 00496 if hasattr(self.process,theModuleLabel): 00497 raise Exception("the current process already has a module "+theModuleLabel+" defined") 00498 #print "creating output module ",theModuleLabel 00499 setattr(self.process,theModuleLabel,output) 00500 outputModule=getattr(self.process,theModuleLabel) 00501 setattr(self.process,theModuleLabel+'_step',cms.EndPath(outputModule)) 00502 path=getattr(self.process,theModuleLabel+'_step') 00503 self.schedule.append(path) 00504 00505 if not self._options.inlineEventContent and hasattr(self.process,theStreamType+"EventContent"): 00506 def doNotInlineEventContent(instance,label = "cms.untracked.vstring(process."+theStreamType+"EventContent.outputCommands)"): 00507 return label 00508 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 00509 if theExtraOutputCommands: 00510 if not isinstance(theExtraOutputCommands,list): 00511 raise Exception("extra ouput command in --option must be a list of strings") 00512 if hasattr(self.process,theStreamType+"EventContent"): 00513 self.executeAndRemember('process.%s.outputCommands.extend(%s)'%(theModuleLabel,theExtraOutputCommands)) 00514 else: 00515 outputModule.outputCommands.extend(theExtraOutputCommands) 00516 00517 result+="\nprocess."+theModuleLabel+" = "+outputModule.dumpPython() 00518 00519 ##ends the --output options model 00520 return result 00521 00522 streamTypes=self._options.eventcontent.split(',') 00523 tiers=self._options.datatier.split(',') 00524 if not self._options.outputDefinition and len(streamTypes)!=len(tiers): 00525 raise Exception("number of event content arguments does not match number of datatier arguments") 00526 00527 # if the only step is alca we don't need to put in an output 00528 if self._options.step.split(',')[0].split(':')[0] == 'ALCA': 00529 return "\n" 00530 00531 for i,(streamType,tier) in enumerate(zip(streamTypes,tiers)): 00532 if streamType=='': continue 00533 if streamType=='DQMROOT': streamType='DQM' 00534 theEventContent = getattr(self.process, streamType+"EventContent") 00535 if i==0: 00536 theFileName=self._options.outfile_name 00537 theFilterName=self._options.filtername 00538 else: 00539 theFileName=self._options.outfile_name.replace('.root','_in'+streamType+'.root') 00540 theFilterName=self._options.filtername 00541 CppType='PoolOutputModule' 00542 if streamType=='DQM' and tier=='DQMROOT': CppType='DQMRootOutputModule' 00543 output = cms.OutputModule(CppType, 00544 theEventContent, 00545 fileName = cms.untracked.string(theFileName), 00546 dataset = cms.untracked.PSet(dataTier = cms.untracked.string(tier), 00547 filterName = cms.untracked.string(theFilterName) 00548 ) 00549 ) 00550 if hasattr(self.process,"generation_step"): 00551 output.SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring('generation_step')) 00552 00553 if streamType=='ALCARECO': 00554 output.dataset.filterName = cms.untracked.string('StreamALCACombined') 00555 00556 outputModuleName=streamType+'output' 00557 setattr(self.process,outputModuleName,output) 00558 outputModule=getattr(self.process,outputModuleName) 00559 setattr(self.process,outputModuleName+'_step',cms.EndPath(outputModule)) 00560 path=getattr(self.process,outputModuleName+'_step') 00561 self.schedule.append(path) 00562 00563 if not self._options.inlineEventContent: 00564 def doNotInlineEventContent(instance,label = "process."+streamType+"EventContent.outputCommands"): 00565 return label 00566 outputModule.outputCommands.__dict__["dumpPython"] = doNotInlineEventContent 00567 00568 result+="\nprocess."+outputModuleName+" = "+outputModule.dumpPython() 00569 00570 return result
| def ConfigBuilder::ConfigBuilder::addSource | ( | self | ) |
Here the source is built. Priority: file, generator
Definition at line 303 of file ConfigBuilder.py.
00304 : 00305 """Here the source is built. Priority: file, generator""" 00306 self.addedObjects.append(("Input source","source")) 00307 00308 def filesFromOption(self): 00309 for entry in self._options.filein.split(','): 00310 print "entry",entry 00311 if entry.startswith("filelist:"): 00312 filesFromList(entry[9:],self.process.source) 00313 elif entry.startswith("dbs:"): 00314 filesFromDBSQuery('find file where dataset = %s'%(entry[4:]),self.process.source) 00315 else: 00316 self.process.source.fileNames.append(self._options.dirin+entry) 00317 if self._options.secondfilein: 00318 if not hasattr(self.process.source,"secondaryFileNames"): 00319 raise Exception("--secondfilein not compatible with "+self._options.filetype+"input type") 00320 for entry in self._options.secondfilein.split(','): 00321 print "entry",entry 00322 if entry.startswith("filelist:"): 00323 self.process.source.secondaryFileNames.extend((filesFromList(entry[9:]))[0]) 00324 elif entry.startswith("dbs:"): 00325 self.process.source.secondaryFileNames.extend((filesFromDBSQuery('find file where dataset = %s'%(entry[4:])))[0]) 00326 else: 00327 self.process.source.secondaryFileNames.append(self._options.dirin+entry) 00328 00329 if self._options.filein: 00330 if self._options.filetype == "EDM": 00331 self.process.source=cms.Source("PoolSource", 00332 fileNames = cms.untracked.vstring(), 00333 secondaryFileNames= cms.untracked.vstring()) 00334 filesFromOption(self) 00335 elif self._options.filetype == "DAT": 00336 self.process.source=cms.Source("NewEventStreamFileReader",fileNames = cms.untracked.vstring()) 00337 filesFromOption(self) 00338 elif self._options.filetype == "LHE": 00339 self.process.source=cms.Source("LHESource", fileNames = cms.untracked.vstring()) 00340 if self._options.filein.startswith("lhe:"): 00341 #list the article directory automatically 00342 args=self._options.filein.split(':') 00343 article=args[1] 00344 print 'LHE input from article ',article 00345 location='/store/lhe/' 00346 import os 00347 textOfFiles=os.popen('cmsLHEtoEOSManager.py -l '+article) 00348 for line in textOfFiles: 00349 for fileName in [x for x in line.split() if '.lhe' in x]: 00350 self.process.source.fileNames.append(location+article+'/'+fileName) 00351 if len(args)>2: 00352 self.process.source.skipEvents = cms.untracked.uint32(int(args[2])) 00353 else: 00354 filesFromOption(self) 00355 00356 00357 elif self._options.filetype == "DQM": 00358 self.process.source=cms.Source("DQMRootSource", 00359 fileNames = cms.untracked.vstring()) 00360 filesFromOption(self) 00361 00362 if ('HARVESTING' in self.stepMap.keys() or 'ALCAHARVEST' in self.stepMap.keys()) and (not self._options.filetype == "DQM"): 00363 self.process.source.processingMode = cms.untracked.string("RunsAndLumis") 00364 00365 if self._options.dbsquery!='': 00366 self.process.source=cms.Source("PoolSource", fileNames = cms.untracked.vstring(),secondaryFileNames = cms.untracked.vstring()) 00367 filesFromDBSQuery(self._options.dbsquery,self.process.source) 00368 00369 if self._options.inputEventContent: 00370 import copy 00371 def dropSecondDropStar(iec): 00372 #drop occurence of 'drop *' in the list 00373 count=0 00374 for item in iec: 00375 if item=='drop *': 00376 if count!=0: 00377 iec.remove(item) 00378 count+=1 00379 00380 00381 ## allow comma separated input eventcontent 00382 if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring() 00383 for evct in self._options.inputEventContent.split(','): 00384 if evct=='': continue 00385 theEventContent = getattr(self.process, evct+"EventContent") 00386 if hasattr(theEventContent,'outputCommands'): 00387 self.process.source.inputCommands.extend(copy.copy(theEventContent.outputCommands)) 00388 if hasattr(theEventContent,'inputCommands'): 00389 self.process.source.inputCommands.extend(copy.copy(theEventContent.inputCommands)) 00390 00391 dropSecondDropStar(self.process.source.inputCommands) 00392 00393 if not self._options.dropDescendant: 00394 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False) 00395 00396 if self._options.inputCommands: 00397 if not hasattr(self.process.source,'inputCommands'): self.process.source.inputCommands=cms.untracked.vstring() 00398 for command in self._options.inputCommands.split(','): 00399 # remove whitespace around the keep/drop statements 00400 command = command.strip() 00401 if command=='': continue 00402 self.process.source.inputCommands.append(command) 00403 if not self._options.dropDescendant: 00404 self.process.source.dropDescendantsOfDroppedBranches = cms.untracked.bool(False) 00405 00406 00407 if 'GEN' in self.stepMap.keys() or (not self._options.filein and hasattr(self._options, "evt_type")): 00408 if self.process.source is None: 00409 self.process.source=cms.Source("EmptySource") 00410 # if option himix is active, drop possibly duplicate DIGI-RAW info: 00411 if self._options.himix==True: 00412 self.process.source.inputCommands = cms.untracked.vstring('drop *','keep *_generator_*_*','keep *_g4SimHits_*_*') 00413 self.process.source.dropDescendantsOfDroppedBranches=cms.untracked.bool(False) 00414 00415 return
| def ConfigBuilder::ConfigBuilder::addStandardSequences | ( | self | ) |
Add selected standard sequences to the process
Definition at line 571 of file ConfigBuilder.py.
00572 : 00573 """ 00574 Add selected standard sequences to the process 00575 """ 00576 # load the pile up file 00577 if self._options.pileup: 00578 pileupSpec=self._options.pileup.split(',')[0] 00579 from Configuration.StandardSequences.Mixing import Mixing,defineMixing 00580 if not pileupSpec in Mixing and '.' not in pileupSpec and 'file:' not in pileupSpec: 00581 raise Exception(pileupSpec+' is not a know mixing scenario:\n available are: '+'\n'.join(Mixing.keys())) 00582 if '.' in pileupSpec: 00583 mixingDict={'file':pileupSpec} 00584 elif pileupSpec.startswith('file:'): 00585 mixingDict={'file':pileupSpec[5:]} 00586 else: 00587 import copy 00588 mixingDict=copy.copy(Mixing[pileupSpec]) 00589 if len(self._options.pileup.split(','))>1: 00590 mixingDict.update(eval(self._options.pileup[self._options.pileup.find(',')+1:])) 00591 if 'file:' in pileupSpec: 00592 #the file is local 00593 self.process.load(mixingDict['file']) 00594 print "inlining mixing module configuration" 00595 self._options.inlineObjets+=',mix' 00596 else: 00597 self.loadAndRemember(mixingDict['file']) 00598 00599 mixingDict.pop('file') 00600 if self._options.pileup_input: 00601 if self._options.pileup_input.startswith('dbs'): 00602 mixingDict['F']=filesFromDBSQuery('find file where dataset = %s'%(self._options.pileup_input[4:],))[0] 00603 else: 00604 mixingDict['F']=self._options.pileup_input.split(',') 00605 specialization=defineMixing(mixingDict,'FASTSIM' in self.stepMap) 00606 for command in specialization: 00607 self.executeAndRemember(command) 00608 if len(mixingDict)!=0: 00609 raise Exception('unused mixing specification: '+mixingDict.keys().__str__()) 00610 00611 00612 # load the geometry file 00613 try: 00614 if len(self.stepMap): 00615 self.loadAndRemember(self.GeometryCFF) 00616 if 'SIM' in self.stepMap or 'reSIM' in self.stepMap: 00617 self.loadAndRemember(self.SimGeometryCFF) 00618 if self.geometryDBLabel: 00619 self.executeAndRemember('process.XMLFromDBSource.label = cms.string("%s")'%(self.geometryDBLabel)) 00620 except ImportError: 00621 print "Geometry option",self._options.geometry,"unknown." 00622 raise 00623 00624 if len(self.stepMap): 00625 self.loadAndRemember(self.magFieldCFF) 00626 00627 if self._options.restoreRNDSeeds: 00628 self.executeAndRemember('process.RandomNumberGeneratorService.restoreStateLabel=cms.untracked.string("randomEngineStateProducer")') 00629 if not self._options.inputCommands: self._options.inputCommands='keep *_randomEngineStateProducer_*_*,' 00630 else: self._options.inputCommands+='keep *_randomEngineStateProducer_*_*,' 00631 00632 # what steps are provided by this class? 00633 stepList = [re.sub(r'^prepare_', '', methodName) for methodName in ConfigBuilder.__dict__ if methodName.startswith('prepare_')] 00634 00635 ### Benedikt can we add here a check that assure that we are going to generate a correct config file? 00636 ### i.e. the harvesting do not have to include other step...... 00637 00638 # look which steps are requested and invoke the corresponding method 00639 for step in self._options.step.split(","): 00640 if step == "": 00641 continue 00642 print step 00643 if step.startswith('re'): 00644 ##add the corresponding input content 00645 if step[2:] not in self._options.donotDropOnInput: 00646 self._options.inputEventContent='%s,%s'%(step.split(":")[0].upper(),self._options.inputEventContent) 00647 step=step[2:] 00648 stepParts = step.split(":") # for format STEP:alternativeSequence 00649 stepName = stepParts[0] 00650 if stepName not in stepList: 00651 raise ValueError("Step "+stepName+" unknown") 00652 if len(stepParts)==1: 00653 getattr(self,"prepare_"+step)(sequence = getattr(self,step+"DefaultSeq")) 00654 elif len(stepParts)==2: 00655 getattr(self,"prepare_"+stepName)(sequence = stepParts[1]) 00656 elif len(stepParts)==3: 00657 getattr(self,"prepare_"+stepName)(sequence = stepParts[1]+','+stepParts[2]) 00658 00659 else: 00660 raise ValueError("Step definition "+step+" invalid")
| def ConfigBuilder::ConfigBuilder::build_production_info | ( | self, | |
| evt_type, | |||
| evtnumber | |||
| ) |
Add useful info for the production.
Definition at line 1759 of file ConfigBuilder.py.
01760 : 01761 """ Add useful info for the production. """ 01762 self.process.configurationMetadata=cms.untracked.PSet\ 01763 (version=cms.untracked.string("$Revision: 1.381 $"), 01764 name=cms.untracked.string("PyReleaseValidation"), 01765 annotation=cms.untracked.string(evt_type+ " nevts:"+str(evtnumber)) 01766 ) 01767 01768 self.addedObjects.append(("Production Info","configurationMetadata")) 01769
| def ConfigBuilder::ConfigBuilder::define_Configs | ( | self | ) |
Definition at line 813 of file ConfigBuilder.py.
00814 : 00815 if len(self.stepMap): 00816 self.loadAndRemember('Configuration/StandardSequences/Services_cff') 00817 if self._options.particleTable not in defaultOptions.particleTableList: 00818 print 'Invalid particle table provided. Options are:' 00819 print defaultOptions.particleTable 00820 sys.exit(-1) 00821 else: 00822 if len(self.stepMap): 00823 self.loadAndRemember('SimGeneral.HepPDTESSource.'+self._options.particleTable+'_cfi') 00824 00825 self.loadAndRemember('FWCore/MessageService/MessageLogger_cfi') 00826 00827 self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreams_cff" 00828 self.GENDefaultCFF="Configuration/StandardSequences/Generator_cff" 00829 self.SIMDefaultCFF="Configuration/StandardSequences/Sim_cff" 00830 self.DIGIDefaultCFF="Configuration/StandardSequences/Digi_cff" 00831 self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRaw_cff" 00832 self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1Emulator_cff' 00833 self.L1MENUDefaultCFF="Configuration/StandardSequences/L1TriggerDefaultMenu_cff" 00834 self.HLTDefaultCFF="Configuration/StandardSequences/HLTtable_cff" 00835 self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_Data_cff" 00836 self.L1RecoDefaultCFF="Configuration/StandardSequences/L1Reco_cff" 00837 self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_Data_cff" 00838 self.SKIMDefaultCFF="Configuration/StandardSequences/Skims_cff" 00839 self.POSTRECODefaultCFF="Configuration/StandardSequences/PostRecoGenerator_cff" 00840 self.VALIDATIONDefaultCFF="Configuration/StandardSequences/Validation_cff" 00841 self.L1HwValDefaultCFF = "Configuration/StandardSequences/L1HwVal_cff" 00842 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOffline_cff" 00843 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/Harvesting_cff" 00844 self.ALCAHARVESTDefaultCFF="Configuration/StandardSequences/AlCaHarvesting_cff" 00845 self.ENDJOBDefaultCFF="Configuration/StandardSequences/EndOfProcess_cff" 00846 self.ConditionsDefaultCFF = "Configuration/StandardSequences/FrontierConditions_GlobalTag_cff" 00847 self.CFWRITERDefaultCFF = "Configuration/StandardSequences/CrossingFrameWriter_cff" 00848 self.REPACKDefaultCFF="Configuration/StandardSequences/DigiToRaw_Repack_cff" 00849 00850 if "DATAMIX" in self.stepMap.keys(): 00851 self.DATAMIXDefaultCFF="Configuration/StandardSequences/DataMixer"+self._options.datamix+"_cff" 00852 self.DIGIDefaultCFF="Configuration/StandardSequences/DigiDM_cff" 00853 self.DIGI2RAWDefaultCFF="Configuration/StandardSequences/DigiToRawDM_cff" 00854 self.L1EMDefaultCFF='Configuration/StandardSequences/SimL1EmulatorDM_cff' 00855 00856 self.ALCADefaultSeq=None 00857 self.SIMDefaultSeq=None 00858 self.GENDefaultSeq='pgen' 00859 self.DIGIDefaultSeq='pdigi' 00860 self.DATAMIXDefaultSeq=None 00861 self.DIGI2RAWDefaultSeq='DigiToRaw' 00862 self.HLTDefaultSeq='GRun' 00863 self.L1DefaultSeq=None 00864 self.L1REPACKDefaultSeq='GT' 00865 self.HARVESTINGDefaultSeq=None 00866 self.ALCAHARVESTDefaultSeq=None 00867 self.CFWRITERDefaultSeq=None 00868 self.RAW2DIGIDefaultSeq='RawToDigi' 00869 self.L1RecoDefaultSeq='L1Reco' 00870 if 'RAW2DIGI' in self.stepMap and 'RECO' in self.stepMap: 00871 self.RECODefaultSeq='reconstruction' 00872 else: 00873 self.RECODefaultSeq='reconstruction_fromRECO' 00874 00875 self.POSTRECODefaultSeq=None 00876 self.L1HwValDefaultSeq='L1HwVal' 00877 self.DQMDefaultSeq='DQMOffline' 00878 self.FASTSIMDefaultSeq='all' 00879 self.VALIDATIONDefaultSeq='' 00880 self.PATLayer0DefaultSeq='all' 00881 self.ENDJOBDefaultSeq='endOfProcess' 00882 self.REPACKDefaultSeq='DigiToRawRepack' 00883 00884 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContent_cff" 00885 00886 # if fastsim switch event content 00887 if "FASTSIM" in self.stepMap.keys(): 00888 self.GENDefaultSeq='pgen_genonly' 00889 self.EVTCONTDefaultCFF = "FastSimulation/Configuration/EventContent_cff" 00890 self.VALIDATIONDefaultCFF = "FastSimulation.Configuration.Validation_cff" 00891 00892 if not self._options.beamspot: 00893 self._options.beamspot=VtxSmearedDefaultKey 00894 00895 # if its MC then change the raw2digi 00896 if self._options.isMC==True: 00897 self.RAW2DIGIDefaultCFF="Configuration/StandardSequences/RawToDigi_cff" 00898 self.RECODefaultCFF="Configuration/StandardSequences/Reconstruction_cff" 00899 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineMC_cff" 00900 self.ALCADefaultCFF="Configuration/StandardSequences/AlCaRecoStreamsMC_cff" 00901 else: 00902 self._options.beamspot = None 00903 00904 #patch for gen, due to backward incompatibility 00905 if 'reGEN' in self.stepMap: 00906 self.GENDefaultSeq='fixGenInfo' 00907 00908 if self._options.scenario=='nocoll' or self._options.scenario=='cosmics': 00909 self.SIMDefaultCFF="Configuration/StandardSequences/SimNOBEAM_cff" 00910 self._options.beamspot='NoSmear' 00911 00912 if self._options.scenario=='cosmics': 00913 self.DIGIDefaultCFF="Configuration/StandardSequences/DigiCosmics_cff" 00914 self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionCosmics_cff" 00915 self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsCosmics_cff" 00916 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentCosmics_cff" 00917 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmics_cff" 00918 if self._options.isMC==True: 00919 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineCosmicsMC_cff" 00920 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingCosmics_cff" 00921 self.RECODefaultSeq='reconstructionCosmics' 00922 self.DQMDefaultSeq='DQMOfflineCosmics' 00923 00924 if self._options.himix: 00925 print "From the presence of the himix option, we have determined that this is heavy ions and will use '--scenario HeavyIons'." 00926 self._options.scenario='HeavyIons' 00927 00928 if self._options.scenario=='HeavyIons': 00929 if not self._options.beamspot: 00930 self._options.beamspot=VtxSmearedHIDefaultKey 00931 self.HLTDefaultSeq = 'HIon' 00932 if not self._options.himix: 00933 self.GENDefaultSeq='pgen_hi' 00934 else: 00935 self.GENDefaultSeq='pgen_himix' 00936 self.VALIDATIONDefaultCFF="Configuration/StandardSequences/ValidationHeavyIons_cff" 00937 self.VALIDATIONDefaultSeq='' 00938 self.EVTCONTDefaultCFF="Configuration/EventContent/EventContentHeavyIons_cff" 00939 self.RECODefaultCFF="Configuration/StandardSequences/ReconstructionHeavyIons_cff" 00940 self.RECODefaultSeq='reconstructionHeavyIons' 00941 self.ALCADefaultCFF = "Configuration/StandardSequences/AlCaRecoStreamsHeavyIons_cff" 00942 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIons_cff" 00943 self.DQMDefaultSeq='DQMOfflineHeavyIons' 00944 self.SKIMDefaultCFF="Configuration/StandardSequences/SkimsHeavyIons_cff" 00945 self.HARVESTINGDefaultCFF="Configuration/StandardSequences/HarvestingHeavyIons_cff" 00946 if self._options.isMC==True: 00947 self.DQMOFFLINEDefaultCFF="DQMOffline/Configuration/DQMOfflineHeavyIonsMC_cff" 00948 00949 00950 self.RAW2RECODefaultSeq=','.join([self.RAW2DIGIDefaultSeq,self.RECODefaultSeq]) 00951 00952 self.USERDefaultSeq='user' 00953 self.USERDefaultCFF=None 00954 00955 # the magnetic field 00956 if self._options.isData: 00957 if self._options.magField==defaultOptions.magField: 00958 print "magnetic field option forced to: AutoFromDBCurrent" 00959 self._options.magField='AutoFromDBCurrent' 00960 self.magFieldCFF = 'Configuration/StandardSequences/MagneticField_'+self._options.magField.replace('.','')+'_cff' 00961 self.magFieldCFF = self.magFieldCFF.replace("__",'_') 00962 00963 # the geometry 00964 self.GeometryCFF='Configuration/StandardSequences/GeometryRecoDB_cff' 00965 self.geometryDBLabel=None 00966 simGeometry='' 00967 if 'FASTSIM' in self.stepMap: 00968 if 'start' in self._options.conditions.lower(): 00969 self.GeometryCFF='FastSimulation/Configuration/Geometries_START_cff' 00970 else: 00971 self.GeometryCFF='FastSimulation/Configuration/Geometries_MC_cff' 00972 else: 00973 def inGeometryKeys(opt): 00974 from Configuration.StandardSequences.GeometryConf import GeometryConf 00975 if opt in GeometryConf: 00976 return GeometryConf[opt] 00977 else: 00978 return opt 00979 00980 geoms=self._options.geometry.split(',') 00981 if len(geoms)==1: geoms=inGeometryKeys(geoms[0]).split(',') 00982 if len(geoms)==2: 00983 #may specify the reco geometry 00984 if '/' in geoms[1] or '_cff' in geoms[1]: 00985 self.GeometryCFF=geoms[1] 00986 else: 00987 self.GeometryCFF='Configuration/StandardSequences/Geometry'+geoms[1]+'_cff' 00988 00989 if (geoms[0].startswith('DB:')): 00990 self.SimGeometryCFF='Configuration/StandardSequences/GeometrySimDB_cff' 00991 self.geometryDBLabel=geoms[0][3:] 00992 print "with DB:" 00993 else: 00994 if '/' in geoms[0] or '_cff' in geoms[0]: 00995 self.SimGeometryCFF=geoms[0] 00996 else: 00997 simGeometry=geoms[0] 00998 if self._options.gflash==True: 00999 self.SimGeometryCFF='Configuration/StandardSequences/Geometry'+geoms[0]+'GFlash_cff' 01000 else: 01001 self.SimGeometryCFF='Configuration/StandardSequences/Geometry'+geoms[0]+'_cff' 01002 01003 # synchronize the geometry configuration and the FullSimulation sequence to be used 01004 if simGeometry not in defaultOptions.geometryExtendedOptions: 01005 self.SIMDefaultCFF="Configuration/StandardSequences/SimIdeal_cff" 01006 01007 # Mixing 01008 if self._options.pileup=='default': 01009 from Configuration.StandardSequences.Mixing import MixingDefaultKey,MixingFSDefaultKey 01010 if 'FASTSIM' in self.stepMap: 01011 self._options.pileup=MixingFSDefaultKey 01012 else: 01013 self._options.pileup=MixingDefaultKey 01014 01015 #not driven by a default cff anymore 01016 if self._options.isData: 01017 self._options.pileup=None 01018 if self._options.isMC==True and self._options.himix==False: 01019 if 'FASTSIM' in self.stepMap: 01020 self._options.pileup='FS_'+self._options.pileup 01021 elif self._options.isMC==True and self._options.himix==True: 01022 self._options.pileup='HiMix' 01023 01024 01025 if self._options.slhc: 01026 if 'stdgeom' not in self._options.slhc: 01027 self.SimGeometryCFF='SLHCUpgradeSimulations.Geometry.%s_cmsSimIdealGeometryXML_cff'%(self._options.slhc,) 01028 self.DIGIDefaultCFF='SLHCUpgradeSimulations/Geometry/Digi_%s_cff'%(self._options.slhc,) 01029 if self._options.pileup!=defaultOptions.pileup: 01030 self._options.pileup='SLHC_%s_%s'%(self._options.pileup,self._options.slhc) 01031 01032 self.REDIGIDefaultSeq=self.DIGIDefaultSeq
| def ConfigBuilder::ConfigBuilder::executeAndRemember | ( | self, | |
| command | |||
| ) |
helper routine to remember replace statements
Definition at line 263 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::expandMapping | ( | self, | |
| seqList, | |||
| mapping, | |||
index = None |
|||
| ) |
Definition at line 1599 of file ConfigBuilder.py.
01600 : 01601 maxLevel=20 01602 level=0 01603 while '@' in repr(seqList) and level<maxLevel: 01604 level+=1 01605 for specifiedCommand in seqList: 01606 if specifiedCommand.startswith('@'): 01607 location=specifiedCommand[1:] 01608 if not location in mapping: 01609 raise Exception("Impossible to map "+location+" from "+repr(mapping)) 01610 mappedTo=mapping[location] 01611 if index!=None: 01612 mappedTo=mappedTo[index] 01613 seqList.remove(specifiedCommand) 01614 seqList.extend(mappedTo.split('+')) 01615 break; 01616 if level==maxLevel: 01617 raise Exception("Could not fully expand "+repr(seqList)+" from "+repr(mapping))
| def ConfigBuilder::ConfigBuilder::finalizeFastSimHLT | ( | self | ) |
Definition at line 1710 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::loadAndRemember | ( | self, | |
| includeFile | |||
| ) |
helper routine to load am memorize imports
Definition at line 254 of file ConfigBuilder.py.
00255 : 00256 """helper routine to load am memorize imports""" 00257 # we could make the imports a on-the-fly data method of the process instance itself 00258 # not sure if the latter is a good idea 00259 includeFile = includeFile.replace('/','.') 00260 self.imports.append(includeFile) 00261 self.process.load(includeFile) 00262 return sys.modules[includeFile]
| def ConfigBuilder::ConfigBuilder::loadDefaultOrSpecifiedCFF | ( | self, | |
| sequence, | |||
| defaultCFF | |||
| ) |
Definition at line 1099 of file ConfigBuilder.py.
01100 : 01101 if ( len(sequence.split('.'))==1 ): 01102 l=self.loadAndRemember(defaultCFF) 01103 elif ( len(sequence.split('.'))==2 ): 01104 l=self.loadAndRemember(sequence.split('.')[0]) 01105 sequence=sequence.split('.')[1] 01106 else: 01107 print "sub sequence configuration must be of the form dir/subdir/cff.a+b+c or cff.a" 01108 print sequence,"not recognized" 01109 raise 01110 return l
| def ConfigBuilder::ConfigBuilder::prepare | ( | self, | |
doChecking = False |
|||
| ) |
Prepare the configuration string and add missing pieces.
Definition at line 1770 of file ConfigBuilder.py.
01771 : 01772 """ Prepare the configuration string and add missing pieces.""" 01773 01774 self.loadAndRemember(self.EVTCONTDefaultCFF) #load the event contents regardless 01775 self.addMaxEvents() 01776 self.addStandardSequences() 01777 if self.with_input: 01778 self.addSource() 01779 self.addConditions() 01780 01781 01782 outputModuleCfgCode="" 01783 if not 'HARVESTING' in self.stepMap.keys() and not 'SKIM' in self.stepMap.keys() and not 'ALCAHARVEST' in self.stepMap.keys() and not 'ALCAOUTPUT' in self.stepMap.keys() and self.with_output: 01784 outputModuleCfgCode=self.addOutput() 01785 01786 self.addCommon() 01787 01788 self.pythonCfgCode = "# Auto generated configuration file\n" 01789 self.pythonCfgCode += "# using: \n# "+__version__[1:-1]+"\n# "+__source__[1:-1]+'\n' 01790 self.pythonCfgCode += "# with command line options: "+self._options.arguments+'\n' 01791 self.pythonCfgCode += "import FWCore.ParameterSet.Config as cms\n\n" 01792 self.pythonCfgCode += "process = cms.Process('"+self.process.name_()+"')\n\n" 01793 01794 self.pythonCfgCode += "# import of standard configurations\n" 01795 for module in self.imports: 01796 self.pythonCfgCode += ("process.load('"+module+"')\n") 01797 01798 # production info 01799 if not hasattr(self.process,"configurationMetadata"): 01800 self.build_production_info(self._options.evt_type, self._options.number) 01801 else: 01802 #the PSet was added via a load 01803 self.addedObjects.append(("Production Info","configurationMetadata")) 01804 01805 self.pythonCfgCode +="\n" 01806 for comment,object in self.addedObjects: 01807 if comment!="": 01808 self.pythonCfgCode += "\n# "+comment+"\n" 01809 self.pythonCfgCode += dumpPython(self.process,object) 01810 01811 # dump the output definition 01812 self.pythonCfgCode += "\n# Output definition\n" 01813 self.pythonCfgCode += outputModuleCfgCode 01814 01815 # dump all additional outputs (e.g. alca or skim streams) 01816 self.pythonCfgCode += "\n# Additional output definition\n" 01817 #I do not understand why the keys are not normally ordered. 01818 nl=self.additionalOutputs.keys() 01819 nl.sort() 01820 for name in nl: 01821 output = self.additionalOutputs[name] 01822 self.pythonCfgCode += "process.%s = %s" %(name, output.dumpPython()) 01823 tmpOut = cms.EndPath(output) 01824 setattr(self.process,name+'OutPath',tmpOut) 01825 self.schedule.append(tmpOut) 01826 01827 # dump all additional commands 01828 self.pythonCfgCode += "\n# Other statements\n" 01829 for command in self.additionalCommands: 01830 self.pythonCfgCode += command + "\n" 01831 01832 #comma separated list of objects that deserve to be inlined in the configuration (typically from a modified config deep down) 01833 for object in self._options.inlineObjets.split(','): 01834 if not object: 01835 continue 01836 if not hasattr(self.process,object): 01837 print 'cannot inline -'+object+'- : not known' 01838 else: 01839 self.pythonCfgCode +='\n' 01840 self.pythonCfgCode +=dumpPython(self.process,object) 01841 01842 # dump all paths 01843 self.pythonCfgCode += "\n# Path and EndPath definitions\n" 01844 for path in self.process.paths: 01845 if getattr(self.process,path) not in self.blacklist_paths: 01846 self.pythonCfgCode += dumpPython(self.process,path) 01847 01848 for endpath in self.process.endpaths: 01849 if getattr(self.process,endpath) not in self.blacklist_paths: 01850 self.pythonCfgCode += dumpPython(self.process,endpath) 01851 01852 # dump the schedule 01853 self.pythonCfgCode += "\n# Schedule definition\n" 01854 result = "process.schedule = cms.Schedule(" 01855 01856 # handling of the schedule 01857 self.process.schedule = cms.Schedule() 01858 for item in self.schedule: 01859 if not isinstance(item, cms.Schedule): 01860 self.process.schedule.append(item) 01861 else: 01862 self.process.schedule.extend(item) 01863 01864 if hasattr(self.process,"HLTSchedule"): 01865 beforeHLT = self.schedule[:self.schedule.index(self.process.HLTSchedule)] 01866 afterHLT = self.schedule[self.schedule.index(self.process.HLTSchedule)+1:] 01867 pathNames = ['process.'+p.label_() for p in beforeHLT] 01868 result += ','.join(pathNames)+')\n' 01869 result += 'process.schedule.extend(process.HLTSchedule)\n' 01870 pathNames = ['process.'+p.label_() for p in afterHLT] 01871 result += 'process.schedule.extend(['+','.join(pathNames)+'])\n' 01872 else: 01873 pathNames = ['process.'+p.label_() for p in self.schedule] 01874 result ='process.schedule = cms.Schedule('+','.join(pathNames)+')\n' 01875 01876 self.pythonCfgCode += result 01877 01878 #repacked version 01879 if self._options.isRepacked: 01880 self.pythonCfgCode +="\n" 01881 self.pythonCfgCode +="from Configuration.PyReleaseValidation.ConfigBuilder import MassReplaceInputTag\n" 01882 self.pythonCfgCode +="MassReplaceInputTag(process)\n" 01883 MassReplaceInputTag(self.process) 01884 01885 # special treatment in case of production filter sequence 2/2 01886 if self.productionFilterSequence: 01887 self.pythonCfgCode +='# filter all path with the production filter sequence\n' 01888 self.pythonCfgCode +='for path in process.paths:\n' 01889 self.pythonCfgCode +='\tgetattr(process,path)._seq = process.%s * getattr(process,path)._seq \n'%(self.productionFilterSequence,) 01890 pfs = getattr(self.process,self.productionFilterSequence) 01891 for path in self.process.paths: 01892 getattr(self.process,path)._seq = pfs * getattr(self.process,path)._seq 01893 01894 # dump customise fragment 01895 self.pythonCfgCode += self.addCustomise() 01896 01897 # make the .io file 01898 01899 if self._options.io: 01900 #io=open(self._options.python_filename.replace('.py','.io'),'w') 01901 if not self._options.io.endswith('.io'): self._option.io+='.io' 01902 io=open(self._options.io,'w') 01903 ioJson={} 01904 if hasattr(self.process.source,"fileNames"): 01905 if len(self.process.source.fileNames.value()): 01906 ioJson['primary']=self.process.source.fileNames.value() 01907 if hasattr(self.process.source,"secondaryFileNames"): 01908 if len(self.process.source.secondaryFileNames.value()): 01909 ioJson['secondary']=self.process.source.secondaryFileNames.value() 01910 01911 for (o,om) in self.process.outputModules_().items(): 01912 ioJson[o]=om.fileName.value() 01913 ioJson['GT']=self._options.conditions 01914 import json 01915 io.write(json.dumps(ioJson)) 01916 return 01917 01918 01919
| def ConfigBuilder::ConfigBuilder::prepare_ALCA | ( | self, | |
sequence = None, |
|||
workflow = 'full' |
|||
| ) |
Enrich the process with alca streams
Definition at line 1144 of file ConfigBuilder.py.
01145 : 01146 """ Enrich the process with alca streams """ 01147 alcaConfig=self.loadDefaultOrSpecifiedCFF(sequence,self.ALCADefaultCFF) 01148 sequence = sequence.split('.')[-1] 01149 01150 # decide which ALCA paths to use 01151 alcaList = sequence.split("+") 01152 maxLevel=0 01153 from Configuration.AlCa.autoAlca import autoAlca 01154 # support @X from autoAlca.py, and recursion support: i.e T0:@Mu+@EG+... 01155 self.expandMapping(alcaList,autoAlca) 01156 01157 for name in alcaConfig.__dict__: 01158 alcastream = getattr(alcaConfig,name) 01159 shortName = name.replace('ALCARECOStream','') 01160 if shortName in alcaList and isinstance(alcastream,cms.FilteredStream): 01161 output = self.addExtraStream(name,alcastream, workflow = workflow) 01162 if 'DQM' in alcaList: 01163 if not self._options.inlineEventContent and hasattr(self.process,name): 01164 self.executeAndRemember('process.' + name + '.outputCommands.append("keep *_MEtoEDMConverter_*_*")') 01165 else: 01166 output.outputCommands.append("keep *_MEtoEDMConverter_*_*") 01167 01168 #rename the HLT process name in the alca modules 01169 if self._options.hltProcess or 'HLT' in self.stepMap: 01170 if isinstance(alcastream.paths,tuple): 01171 for path in alcastream.paths: 01172 self.renameHLTprocessInSequence(path.label()) 01173 else: 01174 self.renameHLTprocessInSequence(alcastream.paths.label()) 01175 01176 for i in range(alcaList.count(shortName)): 01177 alcaList.remove(shortName) 01178 01179 # DQM needs a special handling 01180 elif name == 'pathALCARECODQM' and 'DQM' in alcaList: 01181 path = getattr(alcaConfig,name) 01182 self.schedule.append(path) 01183 alcaList.remove('DQM') 01184 01185 if isinstance(alcastream,cms.Path): 01186 #black list the alca path so that they do not appear in the cfg 01187 self.blacklist_paths.append(alcastream) 01188 01189 01190 if len(alcaList) != 0: 01191 available=[] 01192 for name in alcaConfig.__dict__: 01193 alcastream = getattr(alcaConfig,name) 01194 if isinstance(alcastream,cms.FilteredStream): 01195 available.append(name.replace('ALCARECOStream','')) 01196 print "The following alcas could not be found "+str(alcaList) 01197 print "available ",available 01198 #print "verify your configuration, ignoring for now" 01199 raise Exception("The following alcas could not be found "+str(alcaList))
| def ConfigBuilder::ConfigBuilder::prepare_ALCAHARVEST | ( | self, | |
sequence = None |
|||
| ) |
Enrich the process with AlCaHarvesting step
Definition at line 1681 of file ConfigBuilder.py.
01682 : 01683 """ Enrich the process with AlCaHarvesting step """ 01684 harvestingConfig = self.loadAndRemember(self.ALCAHARVESTDefaultCFF) 01685 sequence=sequence.split(".")[-1] 01686 01687 # decide which AlcaHARVESTING paths to use 01688 harvestingList = sequence.split("+") 01689 for name in harvestingConfig.__dict__: 01690 harvestingstream = getattr(harvestingConfig,name) 01691 if name in harvestingList and isinstance(harvestingstream,cms.Path): 01692 self.schedule.append(harvestingstream) 01693 self.executeAndRemember("process.PoolDBOutputService.toPut.append(process.ALCAHARVEST" + name + "_dbOutput)") 01694 self.executeAndRemember("process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVEST" + name + "_metadata)") 01695 harvestingList.remove(name) 01696 # append the common part at the end of the sequence 01697 lastStep = getattr(harvestingConfig,"ALCAHARVESTDQMSaveAndMetadataWriter") 01698 self.schedule.append(lastStep) 01699 01700 if len(harvestingList) != 0 and 'dummyHarvesting' not in harvestingList : 01701 print "The following harvesting could not be found : ", harvestingList 01702 raise Exception("The following harvesting could not be found : "+str(harvestingList)) 01703 01704
| def ConfigBuilder::ConfigBuilder::prepare_ALCAOUTPUT | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1141 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_ALCAPRODUCER | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1138 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_CFWRITER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the crossing frame writer step
Definition at line 1300 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DATAMIX | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the digitisation step
Definition at line 1306 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DIGI | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the digitisation step
Definition at line 1287 of file ConfigBuilder.py.
01288 : 01289 """ Enrich the schedule with the digitisation step""" 01290 self.loadDefaultOrSpecifiedCFF(sequence,self.DIGIDefaultCFF) 01291 01292 if self._options.gflash==True: 01293 self.loadAndRemember("Configuration/StandardSequences/GFlashDIGI_cff") 01294 01295 if self._options.himix==True: 01296 self.loadAndRemember("SimGeneral/MixingModule/himixDIGI_cff") 01297 01298 self.scheduleSequence(sequence.split('.')[-1],'digitisation_step') 01299 return
| def ConfigBuilder::ConfigBuilder::prepare_DIGI2RAW | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1312 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_DQM | ( | self, | |
sequence = 'DQMOffline' |
|||
| ) |
Definition at line 1618 of file ConfigBuilder.py.
01619 : 01620 # this one needs replacement 01621 01622 self.loadDefaultOrSpecifiedCFF(sequence,self.DQMOFFLINEDefaultCFF) 01623 sequenceList=sequence.split('.')[-1].split('+') 01624 from DQMOffline.Configuration.autoDQM import autoDQM 01625 self.expandMapping(sequenceList,autoDQM,index=0) 01626 01627 if len(set(sequenceList))!=len(sequenceList): 01628 sequenceList=list(set(sequenceList)) 01629 print "Duplicate entries for DQM:, using",sequenceList 01630 pathName='dqmoffline_step' 01631 01632 for (i,sequence) in enumerate(sequenceList): 01633 if (i!=0): 01634 pathName='dqmoffline_%d_step'%(i) 01635 01636 if 'HLT' in self.stepMap.keys() or self._options.hltProcess: 01637 self.renameHLTprocessInSequence(sequence) 01638 01639 # if both HLT and DQM are run in the same process, schedule [HLT]DQM in an EndPath 01640 if 'HLT' in self.stepMap.keys(): 01641 # need to put [HLT]DQM in an EndPath, to access the HLT trigger results 01642 setattr(self.process,pathName, cms.EndPath( getattr(self.process, sequence ) ) ) 01643 else: 01644 # schedule DQM as a standard Path 01645 setattr(self.process,pathName, cms.Path( getattr(self.process, sequence) ) ) 01646 self.schedule.append(getattr(self.process,pathName)) 01647
| def ConfigBuilder::ConfigBuilder::prepare_ENDJOB | ( | self, | |
sequence = 'endOfProcess' |
|||
| ) |
Definition at line 1705 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_FASTSIM | ( | self, | |
sequence = "all" |
|||
| ) |
Enrich the schedule with fastsim
Definition at line 1714 of file ConfigBuilder.py.
01715 : 01716 """Enrich the schedule with fastsim""" 01717 self.loadAndRemember("FastSimulation/Configuration/FamosSequences_cff") 01718 01719 if sequence in ('all','allWithHLTFiltering',''): 01720 if not 'HLT' in self.stepMap.keys(): 01721 self.prepare_HLT(sequence=None) 01722 01723 self.executeAndRemember("process.famosSimHits.SimulateCalorimetry = True") 01724 self.executeAndRemember("process.famosSimHits.SimulateTracking = True") 01725 01726 self.executeAndRemember("process.simulation = cms.Sequence(process.simulationWithFamos)") 01727 self.executeAndRemember("process.HLTEndSequence = cms.Sequence(process.reconstructionWithFamos)") 01728 01729 # since we have HLT here, the process should be called HLT 01730 self._options.name = "HLT" 01731 01732 # if we don't want to filter after HLT but simulate everything regardless of what HLT tells, we have to add reconstruction explicitly 01733 if sequence == 'all' and not 'HLT' in self.stepMap.keys(): #(a) 01734 self.finalizeFastSimHLT() 01735 elif sequence == 'famosWithEverything': 01736 self.process.fastsim_step = cms.Path( getattr(self.process, "famosWithEverything") ) 01737 self.schedule.append(self.process.fastsim_step) 01738 01739 # now the additional commands we need to make the config work 01740 self.executeAndRemember("process.VolumeBasedMagneticFieldESProducer.useParametrizedTrackerField = True") 01741 else: 01742 print "FastSim setting", sequence, "unknown." 01743 raise ValueError 01744 01745 if 'Flat' in self._options.beamspot: 01746 beamspotType = 'Flat' 01747 elif 'Gauss' in self._options.beamspot: 01748 beamspotType = 'Gaussian' 01749 else: 01750 beamspotType = 'BetaFunc' 01751 self.loadAndRemember('IOMC.EventVertexGenerators.VtxSmearedParameters_cfi') 01752 beamspotName = 'process.%sVtxSmearingParameters' %(self._options.beamspot) 01753 self.executeAndRemember(beamspotName+'.type = cms.string("%s")'%(beamspotType)) 01754 self.executeAndRemember('process.famosSimHits.VertexGenerator = '+beamspotName) 01755 if hasattr(self.process,'famosPileUp'): 01756 self.executeAndRemember('process.famosPileUp.VertexGenerator = '+beamspotName) 01757 01758
| def ConfigBuilder::ConfigBuilder::prepare_GEN | ( | self, | |
sequence = None |
|||
| ) |
load the fragment of generator configuration
Definition at line 1200 of file ConfigBuilder.py.
01201 : 01202 """ load the fragment of generator configuration """ 01203 loadFailure=False 01204 #remove trailing .py 01205 #support old style .cfi by changing into something.cfi into something_cfi 01206 #remove python/ from the name 01207 loadFragment = self._options.evt_type.replace('.py','',).replace('.','_').replace('python/','') 01208 #standard location of fragments 01209 if not '/' in loadFragment: 01210 loadFragment='Configuration.Generator.'+loadFragment 01211 else: 01212 loadFragment=loadFragment.replace('/','.') 01213 try: 01214 print "Loading generator fragment from",loadFragment 01215 __import__(loadFragment) 01216 except: 01217 loadFailure=True 01218 if self.process.source and self.process.source.type_()=='EmptySource': 01219 raise Exception("Neither gen fragment of input files provided: this is an inconsistent GEN step configuration") 01220 01221 if not loadFailure: 01222 generatorModule=sys.modules[loadFragment] 01223 genModules=generatorModule.__dict__ 01224 if self._options.hideGen: 01225 self.loadAndRemember(loadFragment) 01226 else: 01227 self.process.load(loadFragment) 01228 # expose the objects from that fragment to the configuration 01229 import FWCore.ParameterSet.Modules as cmstypes 01230 for name in genModules: 01231 theObject = getattr(generatorModule,name) 01232 if isinstance(theObject, cmstypes._Module): 01233 self._options.inlineObjets=name+','+self._options.inlineObjets 01234 elif isinstance(theObject, cms.Sequence) or isinstance(theObject, cmstypes.ESProducer): 01235 self._options.inlineObjets+=','+name 01236 01237 if sequence == self.GENDefaultSeq or sequence == 'pgen_genonly': 01238 if 'ProductionFilterSequence' in genModules and ('generator' in genModules or 'hiSignal' in genModules): 01239 self.productionFilterSequence = 'ProductionFilterSequence' 01240 elif 'generator' in genModules: 01241 self.productionFilterSequence = 'generator' 01242 01243 """ Enrich the schedule with the rest of the generation step """ 01244 self.loadDefaultOrSpecifiedCFF(sequence,self.GENDefaultCFF) 01245 genSeqName=sequence.split('.')[-1] 01246 01247 if not 'FASTSIM' in self.stepMap: 01248 try: 01249 from Configuration.StandardSequences.VtxSmeared import VtxSmeared 01250 self.loadAndRemember(VtxSmeared[self._options.beamspot]) 01251 except ImportError: 01252 raise Exception("VertexSmearing type or beamspot "+self._options.beamspot+" unknown.") 01253 01254 if self._options.scenario == 'HeavyIons' and self._options.himix: 01255 self.loadAndRemember("SimGeneral/MixingModule/himixGEN_cff") 01256 01257 self.process.generation_step = cms.Path( getattr(self.process,genSeqName) ) 01258 self.schedule.append(self.process.generation_step) 01259 01260 if 'reGEN' in self.stepMap: 01261 #stop here 01262 return 01263 01264 """ Enrich the schedule with the summary of the filter step """ 01265 #the gen filter in the endpath 01266 self.loadAndRemember("GeneratorInterface/Core/genFilterSummary_cff") 01267 self.scheduleSequenceAtEnd('genFilterSummary','genfiltersummary_step') 01268 return
| def ConfigBuilder::ConfigBuilder::prepare_HARVESTING | ( | self, | |
sequence = None |
|||
| ) |
Enrich the process with harvesting step
Definition at line 1648 of file ConfigBuilder.py.
01649 : 01650 """ Enrich the process with harvesting step """ 01651 self.EDMtoMECFF='Configuration/StandardSequences/EDMtoME'+self._options.harvesting+'_cff' 01652 self.loadAndRemember(self.EDMtoMECFF) 01653 self.scheduleSequence('EDMtoME','edmtome_step') 01654 01655 harvestingConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.HARVESTINGDefaultCFF) 01656 sequence = sequence.split('.')[-1] 01657 01658 # decide which HARVESTING paths to use 01659 harvestingList = sequence.split("+") 01660 from DQMOffline.Configuration.autoDQM import autoDQM 01661 self.expandMapping(harvestingList,autoDQM,index=1) 01662 01663 if len(set(harvestingList))!=len(harvestingList): 01664 harvestingList=list(set(harvestingList)) 01665 print "Duplicate entries for HARVESTING, using",harvestingList 01666 01667 for name in harvestingList: 01668 if not name in harvestingConfig.__dict__: 01669 print name,"is not a possible harvesting type. Available are",harvestingConfig.__dict__.keys() 01670 continue 01671 harvestingstream = getattr(harvestingConfig,name) 01672 if isinstance(harvestingstream,cms.Path): 01673 self.schedule.append(harvestingstream) 01674 self.blacklist_paths.append(harvestingstream) 01675 if isinstance(harvestingstream,cms.Sequence): 01676 setattr(self.process,name+"_step",cms.Path(harvestingstream)) 01677 self.schedule.append(getattr(self.process,name+"_step")) 01678 01679 self.scheduleSequence('DQMSaver','dqmsave_step') 01680 return
| def ConfigBuilder::ConfigBuilder::prepare_HLT | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the HLT simulation step
Definition at line 1347 of file ConfigBuilder.py.
01348 : 01349 """ Enrich the schedule with the HLT simulation step""" 01350 if not sequence: 01351 print "no specification of the hlt menu has been given, should never happen" 01352 raise Exception('no HLT sequence provided') 01353 else: 01354 if ',' in sequence: 01355 #case where HLT:something:something was provided 01356 self.executeAndRemember('import HLTrigger.Configuration.Utilities') 01357 optionsForHLT = {} 01358 if self._options.scenario == 'HeavyIons': 01359 optionsForHLT['type'] = 'HIon' 01360 else: 01361 optionsForHLT['type'] = 'GRun' 01362 optionsForHLTConfig = ', '.join('%s=%s' % (key, repr(val)) for (key, val) in optionsForHLT.iteritems()) 01363 self.executeAndRemember('process.loadHltConfiguration("%s",%s)'%(sequence.replace(',',':'),optionsForHLTConfig)) 01364 else: 01365 if 'FASTSIM' in self.stepMap: 01366 self.loadAndRemember('HLTrigger/Configuration/HLT_%s_Famos_cff' % sequence) 01367 else: 01368 self.loadAndRemember('HLTrigger/Configuration/HLT_%s_cff' % sequence) 01369 01370 if self._options.name!='HLT': 01371 self.additionalCommands.append('from HLTrigger.Configuration.CustomConfigs import ProcessName') 01372 self.additionalCommands.append('process=ProcessName(process)') 01373 from HLTrigger.Configuration.CustomConfigs import ProcessName 01374 self.process=ProcessName(self.process) 01375 01376 self.schedule.append(self.process.HLTSchedule) 01377 [self.blacklist_paths.append(path) for path in self.process.HLTSchedule if isinstance(path,(cms.Path,cms.EndPath))] 01378 if ('FASTSIM' in self.stepMap and 'HLT' in self.stepMap): 01379 self.finalizeFastSimHLT() 01380
| def ConfigBuilder::ConfigBuilder::prepare_L1 | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the L1 simulation step
Definition at line 1322 of file ConfigBuilder.py.
01323 : 01324 """ Enrich the schedule with the L1 simulation step""" 01325 if not sequence: 01326 self.loadAndRemember(self.L1EMDefaultCFF) 01327 else: 01328 # let the L1 package decide for the scenarios available 01329 from L1Trigger.Configuration.ConfigBuilder import getConfigsForScenario 01330 listOfImports = getConfigsForScenario(sequence) 01331 for file in listOfImports: 01332 self.loadAndRemember(file) 01333 self.scheduleSequence('SimL1Emulator','L1simulation_step') 01334 return
| def ConfigBuilder::ConfigBuilder::prepare_L1HwVal | ( | self, | |
sequence = 'L1HwVal' |
|||
| ) |
Enrich the schedule with L1 HW validation
Definition at line 1399 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_L1Reco | ( | self, | |
sequence = "L1Reco" |
|||
| ) |
Enrich the schedule with L1 reconstruction
Definition at line 1406 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_L1REPACK | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection
Definition at line 1335 of file ConfigBuilder.py.
01336 : 01337 """ Enrich the schedule with the L1 simulation step, running the L1 emulator on data unpacked from the RAW collection, and repacking the result in a new RAW collection""" 01338 if sequence is not 'GT': 01339 print 'Running the full L1 emulator is not supported yet' 01340 raise Exception('unsupported feature') 01341 if sequence is 'GT': 01342 self.loadAndRemember('Configuration/StandardSequences/SimL1EmulatorRepack_GT_cff') 01343 if self._options.scenario == 'HeavyIons': 01344 self.renameInputTagsInSequence("SimL1Emulator","rawDataCollector","rawDataRepacker") 01345 self.scheduleSequence('SimL1Emulator','L1simulation_step') 01346
| def ConfigBuilder::ConfigBuilder::prepare_POSTRECO | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the postreco step
Definition at line 1468 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RAW2DIGI | ( | self, | |
sequence = "RawToDigi" |
|||
| ) |
Definition at line 1392 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RAW2RECO | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1381 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_RECO | ( | self, | |
sequence = "reconstruction" |
|||
| ) |
Enrich the schedule with reconstruction
Definition at line 1412 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_REPACK | ( | self, | |
sequence = None |
|||
| ) |
Definition at line 1317 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_SIM | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with the simulation step
Definition at line 1269 of file ConfigBuilder.py.
01270 : 01271 """ Enrich the schedule with the simulation step""" 01272 self.loadAndRemember(self.SIMDefaultCFF) 01273 if self._options.gflash==True: 01274 self.loadAndRemember("Configuration/StandardSequences/GFlashSIM_cff") 01275 01276 if self._options.magField=='0T': 01277 self.executeAndRemember("process.g4SimHits.UseMagneticField = cms.bool(False)") 01278 01279 if self._options.himix==True: 01280 if self._options.geometry in defaultOptions.geometryExtendedOptions: 01281 self.loadAndRemember("SimGeneral/MixingModule/himixSIMExtended_cff") 01282 else: 01283 self.loadAndRemember("SimGeneral/MixingModule/himixSIMIdeal_cff") 01284 01285 self.scheduleSequence('psim','simulation_step') 01286 return
| def ConfigBuilder::ConfigBuilder::prepare_SKIM | ( | self, | |
sequence = "all" |
|||
| ) |
Enrich the schedule with skimming fragments
Definition at line 1418 of file ConfigBuilder.py.
01419 : 01420 ''' Enrich the schedule with skimming fragments''' 01421 skimConfig = self.loadDefaultOrSpecifiedCFF(sequence,self.SKIMDefaultCFF) 01422 sequence = sequence.split('.')[-1] 01423 01424 skimlist=sequence.split('+') 01425 ## support @Mu+DiJet+@Electron configuration via autoSkim.py 01426 from Configuration.Skimming.autoSkim import autoSkim 01427 self.expandMapping(skimlist,autoSkim) 01428 01429 #print "dictionnary for skims:",skimConfig.__dict__ 01430 for skim in skimConfig.__dict__: 01431 skimstream = getattr(skimConfig,skim) 01432 if isinstance(skimstream,cms.Path): 01433 #black list the alca path so that they do not appear in the cfg 01434 self.blacklist_paths.append(skimstream) 01435 if (not isinstance(skimstream,cms.FilteredStream)): 01436 continue 01437 shortname = skim.replace('SKIMStream','') 01438 if (sequence=="all"): 01439 self.addExtraStream(skim,skimstream) 01440 elif (shortname in skimlist): 01441 self.addExtraStream(skim,skimstream) 01442 #add a DQM eventcontent for this guy 01443 if self._options.datatier!="": 01444 self.process.load(self.EVTCONTDefaultCFF) 01445 skimstreamDQM = cms.FilteredStream( 01446 responsible = skimstream.responsible, 01447 name = skimstream.name+'DQM', 01448 paths = skimstream.paths, 01449 selectEvents = skimstream.selectEvents, 01450 content = self._options.datatier+'EventContent', 01451 dataTier = cms.untracked.string(self._options.datatier) 01452 ) 01453 self.addExtraStream(skim+'DQM',skimstreamDQM) 01454 for i in range(skimlist.count(shortname)): 01455 skimlist.remove(shortname) 01456 01457 01458 01459 if (skimlist.__len__()!=0 and sequence!="all"): 01460 print 'WARNING, possible typo with SKIM:'+'+'.join(skimlist) 01461 raise Exception('WARNING, possible typo with SKIM:'+'+'.join(skimlist))
| def ConfigBuilder::ConfigBuilder::prepare_USER | ( | self, | |
sequence = None |
|||
| ) |
Enrich the schedule with a user defined sequence
Definition at line 1462 of file ConfigBuilder.py.
| def ConfigBuilder::ConfigBuilder::prepare_VALIDATION | ( | self, | |
sequence = 'validation' |
|||
| ) |
Definition at line 1475 of file ConfigBuilder.py.
01476 : 01477 self.loadDefaultOrSpecifiedCFF(sequence,self.VALIDATIONDefaultCFF) 01478 #in case VALIDATION:something:somethingelse -> something,somethingelse 01479 sequence=sequence.split('.')[-1] 01480 if sequence.find(',')!=-1: 01481 prevalSeqName=sequence.split(',')[0] 01482 valSeqName=sequence.split(',')[1] 01483 else: 01484 postfix='' 01485 if sequence: 01486 postfix='_'+sequence 01487 prevalSeqName='prevalidation'+postfix 01488 valSeqName='validation'+postfix 01489 if not hasattr(self.process,valSeqName): 01490 prevalSeqName='' 01491 valSeqName=sequence 01492 01493 if not 'DIGI' in self.stepMap and not 'FASTSIM' in self.stepMap: 01494 self.loadAndRemember('Configuration.StandardSequences.ReMixingSeeds_cff') 01495 #rename the HLT process in validation steps 01496 if ('HLT' in self.stepMap and not 'FASTSIM' in self.stepMap) or self._options.hltProcess: 01497 self.renameHLTprocessInSequence(valSeqName) 01498 if prevalSeqName: 01499 self.renameHLTprocessInSequence(prevalSeqName) 01500 01501 if prevalSeqName: 01502 self.process.prevalidation_step = cms.Path( getattr(self.process, prevalSeqName ) ) 01503 self.schedule.append(self.process.prevalidation_step) 01504 if valSeqName.startswith('genvalid'): 01505 self.loadAndRemember("IOMC.RandomEngine.IOMC_cff") 01506 self.process.validation_step = cms.Path( getattr(self.process,valSeqName ) ) 01507 else: 01508 self.process.validation_step = cms.EndPath( getattr(self.process,valSeqName ) ) 01509 self.schedule.append(self.process.validation_step) 01510 01511 if not 'DIGI' in self.stepMap and not 'FASTSIM' in self.stepMap: 01512 self.executeAndRemember("process.mix.playback = True") 01513 01514 return 01515
| def ConfigBuilder::ConfigBuilder::profileOptions | ( | self | ) |
addIgProfService Function to add the igprof profile service so that you can dump in the middle of the run.
Definition at line 211 of file ConfigBuilder.py.
00212 : 00213 """ 00214 addIgProfService 00215 Function to add the igprof profile service so that you can dump in the middle 00216 of the run. 00217 """ 00218 profileOpts = self._options.profile.split(':') 00219 profilerStart = 1 00220 profilerInterval = 100 00221 profilerFormat = None 00222 profilerJobFormat = None 00223 00224 if len(profileOpts): 00225 #type, given as first argument is unused here 00226 profileOpts.pop(0) 00227 if len(profileOpts): 00228 startEvent = profileOpts.pop(0) 00229 if not startEvent.isdigit(): 00230 raise Exception("%s is not a number" % startEvent) 00231 profilerStart = int(startEvent) 00232 if len(profileOpts): 00233 eventInterval = profileOpts.pop(0) 00234 if not eventInterval.isdigit(): 00235 raise Exception("%s is not a number" % eventInterval) 00236 profilerInterval = int(eventInterval) 00237 if len(profileOpts): 00238 profilerFormat = profileOpts.pop(0) 00239 00240 00241 if not profilerFormat: 00242 profilerFormat = "%s___%s___%s___%s___%s___%s___%%I.gz" % (self._options.evt_type.replace("_cfi", ""), 00243 self._options.step, 00244 self._options.pileup, 00245 self._options.conditions, 00246 self._options.datatier, 00247 self._options.profileTypeLabel) 00248 if not profilerJobFormat and profilerFormat.endswith(".gz"): 00249 profilerJobFormat = profilerFormat.replace(".gz", "_EndOfJob.gz") 00250 elif not profilerJobFormat: 00251 profilerJobFormat = profilerFormat + "_EndOfJob.gz" 00252 00253 return (profilerStart,profilerInterval,profilerFormat,profilerJobFormat)
| def ConfigBuilder::ConfigBuilder::renameHLTprocessInSequence | ( | self, | |
| sequence, | |||
proc = None, |
|||
HLTprocess = 'HLT' |
|||
| ) |
Definition at line 1585 of file ConfigBuilder.py.
01586 : 01587 if self._options.hltProcess: 01588 proc=self._options.hltProcess 01589 else: 01590 proc=self.process.name_() 01591 if proc==HLTprocess: return 01592 # look up all module in dqm sequence 01593 print "replacing %s process name - sequence %s will use '%s'" % (HLTprocess,sequence, proc) 01594 getattr(self.process,sequence).visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor(HLTprocess,proc,whitelist = ("subSystemFolder",))) 01595 if 'from Configuration.PyReleaseValidation.ConfigBuilder import ConfigBuilder' not in self.additionalCommands: 01596 self.additionalCommands.append('from Configuration.PyReleaseValidation.ConfigBuilder import ConfigBuilder') 01597 self.additionalCommands.append('process.%s.visit(ConfigBuilder.MassSearchReplaceProcessNameVisitor("%s", "%s", whitelist = ("subSystemFolder",)))'% (sequence,HLTprocess, proc)) 01598
| def ConfigBuilder::ConfigBuilder::renameInputTagsInSequence | ( | self, | |
| sequence, | |||
oldT = "rawDataCollector", |
|||
newT = "rawDataRepacker" |
|||
| ) |
Definition at line 1575 of file ConfigBuilder.py.
01576 : 01577 print "Replacing all InputTag %s => %s"%(oldT,newT) 01578 from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag 01579 massSearchReplaceAnyInputTag(getattr(self.process,sequence),oldT,newT) 01580 loadMe='from PhysicsTools.PatAlgos.tools.helpers import massSearchReplaceAnyInputTag' 01581 if not loadMe in self.additionalCommands: 01582 self.additionalCommands.append(loadMe) 01583 self.additionalCommands.append('massSearchReplaceAnyInputTag(process.%s,"%s","%s",False)'%(sequence,oldT,newT))
| def ConfigBuilder::ConfigBuilder::scheduleSequence | ( | self, | |
| seq, | |||
| prefix, | |||
what = 'Path' |
|||
| ) |
Definition at line 1111 of file ConfigBuilder.py.
01112 : 01113 if '*' in seq: 01114 #create only one path with all sequences in it 01115 for i,s in enumerate(seq.split('*')): 01116 if i==0: 01117 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, s) )) 01118 else: 01119 p=getattr(self.process,prefix) 01120 p+=getattr(self.process, s) 01121 self.schedule.append(getattr(self.process,prefix)) 01122 return 01123 else: 01124 #create as many path as many sequences 01125 if not '+' in seq: 01126 setattr(self.process,prefix,getattr(cms,what)( getattr(self.process, seq) )) 01127 self.schedule.append(getattr(self.process,prefix)) 01128 else: 01129 for i,s in enumerate(seq.split('+')): 01130 sn=prefix+'%d'%(i) 01131 setattr(self.process,sn,getattr(cms,what)( getattr(self.process, s) )) 01132 self.schedule.append(getattr(self.process,sn)) 01133 return
| def ConfigBuilder::ConfigBuilder::scheduleSequenceAtEnd | ( | self, | |
| seq, | |||
| prefix | |||
| ) |
Definition at line 1134 of file ConfigBuilder.py.
ConfigBuilder::ConfigBuilder::_options [private] |
Definition at line 153 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 1648 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
Definition at line 1770 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 813 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
Definition at line 153 of file ConfigBuilder.py.
1.7.3