test
CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
MatrixUtil.py
Go to the documentation of this file.
1 class Matrix(dict):
2  def __setitem__(self,key,value):
3  if key in self:
4  print "ERROR in Matrix"
5  print "overwritting",key,"not allowed"
6  else:
7  self.update({float(key):WF(float(key),value)})
8 
9  def addOverride(self,key,override):
10  self[key].addOverride(override)
11 
12 #the class to collect all possible steps
13 class Steps(dict):
14  def __setitem__(self,key,value):
15  if key in self:
16  print "ERROR in Step"
17  print "overwritting",key,"not allowed"
18  import sys
19  sys.exit(-9)
20  else:
21  self.update({key:value})
22  # make the python file named <step>.py
23  #if not '--python' in value: self[key].update({'--python':'%s.py'%(key,)})
24 
25  def overwrite(self,keypair):
26  value=self[keypair[1]]
27  self.update({keypair[0]:value})
28 
29 class WF(list):
30  def __init__(self,n,l):
31  self.extend(l)
32  self.num=n
33  #the actual steps of this WF
34  self.steps=[]
35  self.overrides={}
36  def addOverride(self,overrides):
37  self.overrides=overrides
38 
39  def interpret(self,stepsDict):
40  for s in self:
41  print 'steps',s,stepsDict[s]
42  steps.append(stepsDict[s])
43 
44 
45 
46 def expandLsInterval(lumis):
47  return range(lumis[0],(lumis[1]+1))
48 
50 jsonFile2015 = findFileInPath("DPGAnalysis/Skims/data/Cert_13TeV_16Dec2015ReReco_Collisions15_25ns_50ns_JSON.txt")
51 jsonFile2016 = findFileInPath("DPGAnalysis/Skims/data/Cert_271036-274240_13TeV_PromptReco_Collisions16_JSON.txt")
52 
53 import json
54 with open(jsonFile2015) as data_file:
55  data_json2015 = json.load(data_file)
56 
57 with open(jsonFile2016) as data_file:
58  data_json2016 = json.load(data_file)
59 
60 # return a portion of the 2015 golden json
61 # LS for a full run by default; otherwise a subset of which you determined the size
62 def selectedLS(list_runs=[],maxNum=-1,l_json=data_json2015):
63  # print "maxNum is %s"%(maxNum)
64  if type(list_runs[0]) !=int:
65  print "ERROR: list_runs must be a list of intergers"
66  return None
67  local_dict = {}
68  ls_count = 0
69 
70  for run in list_runs:
71  if str(run) in l_json.keys():
72  # print "run %s is there"%(run)
73  runNumber = run
74  # print "Doing lumi-section selection for run %s: "%(run)
75  for LSsegment in l_json[str(run)] :
76  # print LSsegment
77  ls_count += (LSsegment[-1] - LSsegment[0] + 1)
78  if (ls_count > maxNum) & (maxNum != -1):
79  break
80  # return local_dict
81  if runNumber in local_dict.keys():
82  local_dict[runNumber].append(LSsegment)
83  else:
84  local_dict[runNumber] = [LSsegment]
85  # print "total LS so far %s - grow %s"%(ls_count,local_dict)
86  #local_dict[runNumber] = [1,2,3]
87  else:
88  print "run %s is NOT present in json %s\n\n"%(run, l_json)
89  # print "++ %s"%(local_dict)
90 
91  if ( len(local_dict.keys()) > 0 ) :
92  return local_dict
93  else :
94  print "No luminosity section interval passed the json and your selection; returning None"
95  return None
96 
97 # print "\n\n\n THIS IS WHAT I RETURN: %s \n\n"%( selectedLS([251244,251251]) )
98 
99 
100 
101 
102 InputInfoNDefault=2000000
103 class InputInfo(object):
104  def __init__(self,dataSet,label='',run=[],ls={},files=1000,events=InputInfoNDefault,split=10,location='CAF',ib_blacklist=None,ib_block=None) :
105  self.run = run
106  self.ls = ls
107  self.files = files
108  self.events = events
109  self.location = location
110  self.label = label
111  self.dataSet = dataSet
112  self.split = split
113  self.ib_blacklist = ib_blacklist
114  self.ib_block = ib_block
115 
116  def das(self, das_options):
117  if len(self.run) is not 0 or self.ls:
118  # take at most 5 queries, to avoid sinking das
119 
120  # do if you have LS queries
121  # command = ";".join(["das_client.py %s --query '%s'" % (das_options, query) for query in self.queries()[:3] ])
122  command = ";".join(["das_client %s --query '%s'" % (das_options, query) for query in self.queries()[:3] ])
123  command = "({0})".format(command)
124  else:
125  command = "das_client %s --query '%s'" % (das_options, self.queries()[0])
126 
127  # Run filter on DAS output
128  if self.ib_blacklist:
129  command += " | grep -E -v "
130  command += " ".join(["-e '{0}'".format(pattern) for pattern in self.ib_blacklist])
131  command += " | sort -u"
132  return command
133 
134  def lumiRanges(self):
135  if len(self.run) != 0:
136  return "echo '{\n"+",".join(('"%d":[[1,268435455]]\n'%(x,) for x in self.run))+"}'"
137  if self.ls :
138  return "echo '{\n"+",".join(('"%d" : %s\n'%( int(x),self.ls[x]) for x in self.ls.keys()))+"}'"
139  return None
140 
141  def queries(self):
142  query_by = "block" if self.ib_block else "dataset"
143  query_source = "{0}#{1}".format(self.dataSet, self.ib_block) if self.ib_block else self.dataSet
144 
145  if self.ls :
146  the_queries = []
147  #for query_run in self.ls.keys():
148  # print "run is %s"%(query_run)
149  # if you have a LS list specified, still query das for the full run (multiple ls queries take forever)
150  # and use step1_lumiRanges.log to run only on LS which respect your selection
151 
152  # DO WE WANT T2_CERN ?
153  return ["file {0}={1} run={2}".format(query_by, query_source, query_run) for query_run in self.ls.keys()]
154  #return ["file {0}={1} run={2} site=T2_CH_CERN".format(query_by, query_source, query_run) for query_run in self.ls.keys()]
155 
156 
157  #
158  #for a_range in self.ls[query_run]:
159  # # print "a_range is %s"%(a_range)
160  # the_queries += ["file {0}={1} run={2} lumi={3} ".format(query_by, query_source, query_run, query_ls) for query_ls in expandLsInterval(a_range) ]
161  #print the_queries
162  return the_queries
163 
164  if len(self.run) is not 0:
165  return ["file {0}={1} run={2} site=T2_CH_CERN".format(query_by, query_source, query_run) for query_run in self.run]
166  #return ["file {0}={1} run={2} ".format(query_by, query_source, query_run) for query_run in self.run]
167  else:
168  return ["file {0}={1} site=T2_CH_CERN".format(query_by, query_source)]
169  #return ["file {0}={1} ".format(query_by, query_source)]
170 
171  def __str__(self):
172  if self.ib_block:
173  return "input from: {0} with run {1}#{2}".format(self.dataSet, self.ib_block, self.run)
174  return "input from: {0} with run {1}".format(self.dataSet, self.run)
175 
176 
177 # merge dictionaries, with prioty on the [0] index
178 def merge(dictlist,TELL=False):
179  import copy
180  last=len(dictlist)-1
181  if TELL: print last,dictlist
182  if last==0:
183  # ONLY ONE ITEM LEFT
184  return copy.copy(dictlist[0])
185  else:
186  reducedlist=dictlist[0:max(0,last-1)]
187  if TELL: print reducedlist
188  # make a copy of the last item
189  d=copy.copy(dictlist[last])
190  # update with the last but one item
191  d.update(dictlist[last-1])
192  # and recursively do the rest
193  reducedlist.append(d)
194  return merge(reducedlist,TELL)
195 
196 def remove(d,key,TELL=False):
197  import copy
198  e = copy.deepcopy(d)
199  if TELL: print "original dict, BEF: %s"%d
200  del e[key]
201  if TELL: print "copy-removed dict, AFT: %s"%e
202  return e
203 
204 
205 #### Standard release validation samples ####
206 
207 stCond={'--conditions':'auto:run1_mc'}
208 def Kby(N,s):
209  return {'--relval':'%s000,%s'%(N,s)}
210 def Mby(N,s):
211  return {'--relval':'%s000000,%s'%(N,s)}
212 
213 def changeRefRelease(steps,listOfPairs):
214  for s in steps:
215  if ('INPUT' in steps[s]):
216  oldD=steps[s]['INPUT'].dataSet
217  for (ref,newRef) in listOfPairs:
218  if ref in oldD:
219  steps[s]['INPUT'].dataSet=oldD.replace(ref,newRef)
220  if '--pileup_input' in steps[s]:
221  for (ref,newRef) in listOfPairs:
222  if ref in steps[s]['--pileup_input']:
223  steps[s]['--pileup_input']=steps[s]['--pileup_input'].replace(ref,newRef)
224 
225 def addForAll(steps,d):
226  for s in steps:
227  steps[s].update(d)
228 
229 
230 def genvalid(fragment,d,suffix='all',fi='',dataSet=''):
231  import copy
232  c=copy.copy(d)
233  if suffix:
234  c['-s']=c['-s'].replace('genvalid','genvalid_'+suffix)
235  if fi:
236  c['--filein']='lhe:%d'%(fi,)
237  if dataSet:
238  c['--filein']='das:%s'%(dataSet,)
239  c['cfg']=fragment
240  return c
241 
242 
Definition: merge.py:1
def selectedLS
Definition: MatrixUtil.py:62
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
def expandLsInterval
Definition: MatrixUtil.py:46
def changeRefRelease
Definition: MatrixUtil.py:213
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def genvalid
Definition: MatrixUtil.py:230
#define update(a, b)
def addForAll
Definition: MatrixUtil.py:225
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run