9 print 'Not injecting to wmagent in --show mode. Need to run the worklfows.'
11 if opt.wmcontrol==
'init':
14 if opt.wmcontrol==
'test':
17 if opt.wmcontrol==
'submit' and opt.nProcs==0:
18 print 'Not injecting to wmagent in -j 0 mode. Need to run the worklfows.'
20 if opt.wmcontrol==
'force':
21 print "This is an expert setting, you'd better know what you're doing"
25 from modules.wma
import upload_to_couch
26 (filePath,labelInCouch,user,group,where) = arguments
27 cacheId=upload_to_couch(filePath,
43 for k
in options.split(
','):
44 if k.startswith(
'dqm:'):
45 self.
dqmgui=k.split(
':',1)[-1]
46 elif k.startswith(
'wma:'):
49 self.
testMode=((mode!=
'submit')
and (mode!=
'force'))
57 self.
wmagent=os.getenv(
'WMAGENT_REQMGR')
63 self.
wmagent =
'cmsweb-testbed.cern.ch'
64 self.
DbsUrl =
"https://"+self.
wmagent+
"/dbs/int/global/DBSReader"
67 self.
dqmgui=
"https://cmsweb.cern.ch/dqm/relval"
72 self.
user = os.getenv(
'USER')
80 if not os.getenv(
'WMCORE_ROOT'):
81 print '\n\twmclient is not setup properly. Will not be able to upload or submit requests.\n'
86 print '\n\tFound wmclient\n'
89 "RequestType" :
"TaskChain",
90 "SubRequestType" :
"RelVal",
91 "RequestPriority": 500000,
92 "Requestor": self.
user,
94 "CMSSWVersion": os.getenv(
'CMSSW_VERSION'),
95 "Campaign": os.getenv(
'CMSSW_VERSION'),
96 "ScramArch": os.getenv(
'SCRAM_ARCH'),
97 "ProcessingVersion": self.
version,
99 "CouchURL": self.
couch,
100 "ConfigCacheURL": self.
couch,
106 "unmergedLFNBase" :
"/store/unmerged",
107 "mergedLFNBase" :
"/store/relval",
108 "dashboardActivity" :
"relval",
111 "SizePerEvent" : 1234,
116 "EnableHarvesting" :
"True",
117 "DQMUploadUrl" : self.
dqmgui,
118 "DQMConfigCacheID" :
None,
124 "ConfigCacheID" :
None,
126 "SplittingAlgo" :
"EventBased",
127 "EventsPerJob" :
None,
128 "RequestNumEvents" :
None,
129 "Seeding" :
"AutomaticSeeding",
130 "PrimaryDataset" :
None,
132 "Multicore" : opt.nThreads,
136 "TaskName" :
"DigiHLT",
137 "ConfigCacheID" :
None,
139 "InputDataset" :
None,
140 "SplittingAlgo" :
"LumiBased",
143 "Multicore" : opt.nThreads,
149 "InputFromOutputModule" :
None,
150 "ConfigCacheID" :
None,
152 "SplittingAlgo" :
"LumiBased",
155 "Multicore" : opt.nThreads,
162 def prepare(self,mReader, directories, mode='init'):
169 wmsplit[
'RECODreHLT']=2
173 wmsplit[
'DIGIUP15_PU50']=1
174 wmsplit[
'RECOUP15_PU50']=1
175 wmsplit[
'DIGIUP15_PU25']=1
176 wmsplit[
'RECOUP15_PU25']=1
177 wmsplit[
'DIGIUP15_PU25HS']=1
178 wmsplit[
'RECOUP15_PU25HS']=1
179 wmsplit[
'DIGIHIMIX']=5
180 wmsplit[
'RECOHIMIX']=5
181 wmsplit[
'RECODSplit']=1
182 wmsplit[
'SingleMuPt10_UP15_ID']=1
183 wmsplit[
'DIGIUP15_ID']=1
184 wmsplit[
'RECOUP15_ID']=1
185 wmsplit[
'TTbar_13_ID']=1
186 wmsplit[
'SingleMuPt10FS_ID']=1
187 wmsplit[
'TTbarFS_ID']=1
188 wmsplit[
'RECODR2_50nsreHLT']=1
189 wmsplit[
'RECODR2_25nsreHLT']=1
190 wmsplit[
'RECODR2_2016reHLT']=5
191 wmsplit[
'RECODR2_2016reHLT_skimSingleMu']=5
192 wmsplit[
'RECODR2_2016reHLT_skimDoubleEG']=5
193 wmsplit[
'RECODR2_2016reHLT_skimMuonEG']=5
194 wmsplit[
'RECODR2_2016reHLT_skimJetHT']=5
195 wmsplit[
'RECODR2_2016reHLT_skimMuOnia']=5
196 wmsplit[
'HLTDR2_50ns']=1
197 wmsplit[
'HLTDR2_25ns']=1
198 wmsplit[
'HLTDR2_2016']=1
199 wmsplit[
'Hadronizer']=1
200 wmsplit[
'DIGIUP15']=5
201 wmsplit[
'RECOUP15']=5
202 wmsplit[
'RECOAODUP15']=5
203 wmsplit[
'DBLMINIAODMCUP15NODQM']=5
209 print "Not set up for step splitting"
213 for (n,dir)
in directories.items():
215 print "inspecting",dir
217 for (x,s)
in mReader.workFlowSteps.items():
227 if len( [step
for step
in s[3]
if "HARVESTGEN" in step] )>0:
228 chainDict[
'TimePerEvent']=0.01
229 thisLabel=thisLabel+
"_gen"
231 if len( [step
for step
in s[3]
if "DBLMINIAODMCUP15NODQM" in step] )>0:
232 thisLabel=thisLabel+
"_dblMiniAOD"
237 if 'INPUT' in step
or (
not isinstance(s[2][index],str)):
238 nextHasDSInput=s[2][index]
246 chainDict[
'nowmTasklist'][-1][
'nowmIO']=json.loads(open(
'%s/%s.io'%(dir,step)).
read())
248 print "Failed to find",
'%s/%s.io'%(dir,step),
".The workflows were probably not run on cfg not created"
251 chainDict[
'nowmTasklist'][-1][
'PrimaryDataset']=
'RelVal'+s[1].
split(
'+')[0]
252 if not '--relval' in s[2][index]:
253 print 'Impossible to create task from scratch without splitting information with --relval'
256 arg=s[2][index].
split()
257 ns=map(int,arg[arg.index(
'--relval')+1].
split(
','))
258 chainDict[
'nowmTasklist'][-1][
'RequestNumEvents'] = ns[0]
259 chainDict[
'nowmTasklist'][-1][
'EventsPerJob'] = ns[1]
260 if 'FASTSIM' in s[2][index]
or '--fast' in s[2][index]:
261 thisLabel+=
'_FastSim'
262 if 'lhe' in s[2][index]
in s[2][index]:
263 chainDict[
'nowmTasklist'][-1][
'LheInputFiles'] =
True
268 chainDict[
'nowmTasklist'][-1][
'nowmIO']=json.loads(open(
'%s/%s.io'%(dir,step)).
read())
270 print "Failed to find",
'%s/%s.io'%(dir,step),
".The workflows were probably not run on cfg not created"
272 chainDict[
'nowmTasklist'][-1][
'InputDataset']=nextHasDSInput.dataSet
273 splitForThisWf=nextHasDSInput.split
274 chainDict[
'nowmTasklist'][-1][
'LumisPerJob']=splitForThisWf
276 chainDict[
'nowmTasklist'][-1][
'LumisPerJob']=wmsplit[step]
278 if len(nextHasDSInput.run):
279 chainDict[
'nowmTasklist'][-1][
'RunWhitelist']=nextHasDSInput.run
280 if len(nextHasDSInput.ls):
281 chainDict[
'nowmTasklist'][-1][
'LumiList']=nextHasDSInput.ls
283 if '--data' in s[2][index]
and nextHasDSInput.label:
284 thisLabel+=
'_RelVal_%s'%nextHasDSInput.label
285 if 'filter' in chainDict[
'nowmTasklist'][-1][
'nowmIO']:
286 print "This has an input DS and a filter sequence: very likely to be the PyQuen sample"
287 processStrPrefix=
'PU_'
288 setPrimaryDs =
'RelVal'+s[1].
split(
'+')[0]
290 chainDict[
'nowmTasklist'][-1][
'PrimaryDataset']=setPrimaryDs
296 chainDict[
'nowmTasklist'][-1][
'nowmIO']=json.loads(open(
'%s/%s.io'%(dir,step)).
read())
298 print "Failed to find",
'%s/%s.io'%(dir,step),
".The workflows were probably not run on cfg not created"
301 chainDict[
'nowmTasklist'][-1][
'LumisPerJob']=splitForThisWf
303 chainDict[
'nowmTasklist'][-1][
'LumisPerJob']=wmsplit[step]
306 if 'Hadronizer' in step:
307 chainDict[
'nowmTasklist'][-1][
'LumisPerJob']=wmsplit[
'Hadronizer']
310 chainDict[
'nowmTasklist'][-1][
'TaskName']=step
312 chainDict[
'nowmTasklist'][-1][
'PrimaryDataset']=setPrimaryDs
313 chainDict[
'nowmTasklist'][-1][
'ConfigCacheID']=
'%s/%s.py'%(dir,step)
314 chainDict[
'nowmTasklist'][-1][
'GlobalTag']=chainDict[
'nowmTasklist'][-1][
'nowmIO'][
'GT']
315 chainDict[
'GlobalTag']=chainDict[
'nowmTasklist'][-1][
'nowmIO'][
'GT']
316 if 'pileup' in chainDict[
'nowmTasklist'][-1][
'nowmIO']:
317 chainDict[
'nowmTasklist'][-1][
'MCPileup']=chainDict[
'nowmTasklist'][-1][
'nowmIO'][
'pileup']
318 if '--pileup ' in s[2][index]:
319 processStrPrefix=
'PU_'
321 processStrPrefix=
'PU25ns_'
323 processStrPrefix=
'PU50ns_'
324 if 'DIGIPREMIX_S2' in s[2][index] :
326 processStrPrefix=
'PUpmx25ns_'
327 elif s[2][index].
split()[ s[2][index].
split().
index(
'--pileup_input')+1 ].
find(
'50ns') > 0 :
328 processStrPrefix=
'PUpmx50ns_'
332 chainDict[
'AcquisitionEra'][step]=chainDict[
'CMSSWVersion']
333 chainDict[
'ProcessingString'][step]=processStrPrefix+chainDict[
'nowmTasklist'][-1][
'GlobalTag'].
replace(
'::All',
'')+thisLabel
336 chainDict[
'nowmTasklist'][-1][
'AcquisitionEra']=chainDict[
'CMSSWVersion']
337 chainDict[
'nowmTasklist'][-1][
'ProcessingString']=processStrPrefix+chainDict[
'nowmTasklist'][-1][
'GlobalTag'].
replace(
'::All',
'')+thisLabel
340 if (
'DBLMINIAODMCUP15NODQM' in step):
341 chainDict[
'nowmTasklist'][-1][
'ProcessingString']=chainDict[
'nowmTasklist'][-1][
'ProcessingString']+
'_miniAOD'
343 if( chainDict[
'nowmTasklist'][-1][
'Multicore'] ):
350 chainDict[
'nowmTasklist'][-1][
'Memory'] = self.
memoryOffset + int( chainDict[
'nowmTasklist'][-1][
'Multicore'] -1 ) * self.
memPerCore
354 chainDict[
'RequestString']=
'RV'+chainDict[
'CMSSWVersion']+s[1].
split(
'+')[0]
355 if processStrPrefix
or thisLabel:
356 chainDict[
'RequestString']+=
'_'+processStrPrefix+thisLabel
365 for i_second
in reversed(range(len(chainDict[
'nowmTasklist']))):
366 t_second=chainDict[
'nowmTasklist'][i_second]
368 if 'primary' in t_second[
'nowmIO']:
370 primary=t_second[
'nowmIO'][
'primary'][0].
replace(
'file:',
'')
371 for i_input
in reversed(range(0,i_second)):
372 t_input=chainDict[
'nowmTasklist'][i_input]
373 for (om,o)
in t_input[
'nowmIO'].
items():
376 t_second[
'InputTask'] = t_input[
'TaskName']
377 t_second[
'InputFromOutputModule'] = om
379 if t_second[
'TaskName'].startswith(
'HARVEST'):
381 chainDict[
'DQMConfigCacheID']=t_second[
'ConfigCacheID']
391 chainDict[
'AcquisitionEra'] = chainDict[
'AcquisitionEra'].
values()[0]
397 if type(i)==int
and i < len(chainDict[
'nowmTasklist']):
398 chainDict[
'nowmTasklist'][i][
'KeepOutput']=
True
399 for (i,t)
in enumerate(chainDict[
'nowmTasklist']):
400 if t[
'TaskName'].startswith(
'HARVEST'):
404 elif t[
'TaskName']
in self.
keep:
408 chainDict[
'Task%d'%(itask)]=t
415 chainDict[
'TaskChain']=itask
417 chainDict.pop(
'nowmTasklist')
424 labelInCouch=self.
label+
'_'+label
425 cacheName=filePath.split(
'/')[-1]
428 print '\tFake upload of',filePath,
'to couch with label',labelInCouch
432 from modules.wma
import upload_to_couch,DATABASE_NAME
434 print '\n\tUnable to find wmcontrol modules. Please include it in your python path\n'
439 print "Not re-uploading",filePath,
"to",where,
"for",label
442 print "Loading",filePath,
"to",where,
"for",label
444 pool = multiprocessing.Pool(1)
445 cacheIds = pool.map( upload_to_couch_oneArg, [(filePath,labelInCouch,self.
user,self.
group,where)] )
446 cacheId = cacheIds[0]
451 for (n,d)
in self.chainDicts.items():
453 if it.startswith(
"Task")
and it!=
'TaskChain':
455 couchID=self.
uploadConf(d[it][
'ConfigCacheID'],
456 str(n)+d[it][
'TaskName'],
459 print d[it][
'ConfigCacheID'],
" uploaded to couchDB for",str(n),
"with ID",couchID
460 d[it][
'ConfigCacheID']=couchID
461 if it ==
'DQMConfigCacheID':
462 couchID=self.
uploadConf(d[
'DQMConfigCacheID'],
466 print d[
'DQMConfigCacheID'],
"uploaded to couchDB for",str(n),
"with ID",couchID
467 d[
'DQMConfigCacheID']=couchID
472 from modules.wma
import makeRequest,approveRequest
473 from wmcontrol
import random_sleep
474 print '\n\tFound wmcontrol\n'
476 print '\n\tUnable to find wmcontrol modules. Please include it in your python path\n'
482 for (n,d)
in self.chainDicts.items():
484 print "Only viewing request",n
485 print pprint.pprint(d)
488 print "For eyes before submitting",n
489 print pprint.pprint(d)
490 print "Submitting",n,
"..........."
491 workFlow=makeRequest(self.
wmagent,d,encodeDict=
True)
492 approveRequest(self.
wmagent,workFlow)
493 print "...........",n,
"submitted"
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
def performInjectionOptionTest
def upload_to_couch_oneArg