test
CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
MatrixUtil.py
Go to the documentation of this file.
1 class Matrix(dict):
2  def __setitem__(self,key,value):
3  if key in self:
4  print "ERROR in Matrix"
5  print "overwritting",key,"not allowed"
6  else:
7  self.update({float(key):WF(float(key),value)})
8 
9  def addOverride(self,key,override):
10  self[key].addOverride(override)
11 
12 #the class to collect all possible steps
13 class Steps(dict):
14  def __setitem__(self,key,value):
15  if key in self:
16  print "ERROR in Step"
17  print "overwritting",key,"not allowed"
18  import sys
19  sys.exit(-9)
20  else:
21  self.update({key:value})
22  # make the python file named <step>.py
23  #if not '--python' in value: self[key].update({'--python':'%s.py'%(key,)})
24 
25  def overwrite(self,keypair):
26  value=self[keypair[1]]
27  self.update({keypair[0]:value})
28 
29 class WF(list):
30  def __init__(self,n,l):
31  self.extend(l)
32  self.num=n
33  #the actual steps of this WF
34  self.steps=[]
35  self.overrides={}
36  def addOverride(self,overrides):
37  self.overrides=overrides
38 
39  def interpret(self,stepsDict):
40  for s in self:
41  print 'steps',s,stepsDict[s]
42  steps.append(stepsDict[s])
43 
44 
45 
46 def expandLsInterval(lumis):
47  return range(lumis[0],(lumis[1]+1))
48 
50 jsonFile2015 = findFileInPath("DPGAnalysis/Skims/data/Cert_13TeV_16Dec2015ReReco_Collisions15_25ns_50ns_JSON.txt")
51 jsonFile2016 = findFileInPath("DPGAnalysis/Skims/data/Cert_271036-274240_13TeV_PromptReco_Collisions16_JSON.txt")
52 
53 import json
54 with open(jsonFile2015) as data_file:
55  data_json2015 = json.load(data_file)
56 
57 with open(jsonFile2016) as data_file:
58  data_json2016 = json.load(data_file)
59 
60 # return a portion of the 2015 golden json
61 # LS for a full run by default; otherwise a subset of which you determined the size
62 def selectedLS(list_runs=[],maxNum=-1,l_json=data_json2015):
63  # print "maxNum is %s"%(maxNum)
64  if type(list_runs[0]) !=int:
65  print "ERROR: list_runs must be a list of intergers"
66  return None
67  local_dict = {}
68  ls_count = 0
69 
70  for run in list_runs:
71  if str(run) in l_json.keys():
72  # print "run %s is there"%(run)
73  runNumber = run
74  # print "Doing lumi-section selection for run %s: "%(run)
75  for LSsegment in l_json[str(run)] :
76  # print LSsegment
77  ls_count += (LSsegment[-1] - LSsegment[0] + 1)
78  if (ls_count > maxNum) & (maxNum != -1):
79  break
80  # return local_dict
81  if runNumber in local_dict.keys():
82  local_dict[runNumber].append(LSsegment)
83  else:
84  local_dict[runNumber] = [LSsegment]
85  # print "total LS so far %s - grow %s"%(ls_count,local_dict)
86  #local_dict[runNumber] = [1,2,3]
87  else:
88  print "run %s is NOT present in json %s\n\n"%(run, l_json)
89  # print "++ %s"%(local_dict)
90 
91  if ( len(local_dict.keys()) > 0 ) :
92  return local_dict
93  else :
94  print "No luminosity section interval passed the json and your selection; returning None"
95  return None
96 
97 # print "\n\n\n THIS IS WHAT I RETURN: %s \n\n"%( selectedLS([251244,251251]) )
98 
99 
100 
101 
102 InputInfoNDefault=2000000
103 class InputInfo(object):
104  def __init__(self,dataSet,label='',run=[],ls={},files=1000,events=InputInfoNDefault,split=10,location='CAF',ib_blacklist=None,ib_block=None) :
105  self.run = run
106  self.ls = ls
107  self.files = files
108  self.events = events
109  self.location = location
110  self.label = label
111  self.dataSet = dataSet
112  self.split = split
113  self.ib_blacklist = ib_blacklist
114  self.ib_block = ib_block
115 
116  def das(self, das_options):
117  if len(self.run) is not 0 or self.ls:
118  # take at most 5 queries, to avoid sinking das
119 
120  # do if you have LS queries
121  # command = ";".join(["das_client.py %s --query '%s'" % (das_options, query) for query in self.queries()[:3] ])
122  command = ";".join(["das_client %s --query '%s'" % (das_options, query) for query in self.queries()[:3] ])
123  command = "({0})".format(command)
124  else:
125  command = "das_client %s --query '%s'" % (das_options, self.queries()[0])
126 
127  # Run filter on DAS output
128  if self.ib_blacklist:
129  command += " | grep -E -v "
130  command += " ".join(["-e '{0}'".format(pattern) for pattern in self.ib_blacklist])
131  command += " | sort -u"
132  return command
133 
134  def lumiRanges(self):
135  if len(self.run) != 0:
136  return "echo '{\n"+",".join(('"%d":[[1,268435455]]\n'%(x,) for x in self.run))+"}'"
137  if self.ls :
138  return "echo '{\n"+",".join(('"%d" : %s\n'%( int(x),self.ls[x]) for x in self.ls.keys()))+"}'"
139  return None
140 
141  def queries(self):
142  query_by = "block" if self.ib_block else "dataset"
143  query_source = "{0}#{1}".format(self.dataSet, self.ib_block) if self.ib_block else self.dataSet
144 
145  if self.ls :
146  the_queries = []
147  #for query_run in self.ls.keys():
148  # print "run is %s"%(query_run)
149  # if you have a LS list specified, still query das for the full run (multiple ls queries take forever)
150  # and use step1_lumiRanges.log to run only on LS which respect your selection
151 
152  # DO WE WANT T2_CERN ?
153  return ["file {0}={1} run={2}".format(query_by, query_source, query_run) for query_run in self.ls.keys()]
154  #return ["file {0}={1} run={2} site=T2_CH_CERN".format(query_by, query_source, query_run) for query_run in self.ls.keys()]
155 
156 
157  #
158  #for a_range in self.ls[query_run]:
159  # # print "a_range is %s"%(a_range)
160  # the_queries += ["file {0}={1} run={2} lumi={3} ".format(query_by, query_source, query_run, query_ls) for query_ls in expandLsInterval(a_range) ]
161  #print the_queries
162  return the_queries
163 
164  if len(self.run) is not 0:
165  return ["file {0}={1} run={2} site=T2_CH_CERN".format(query_by, query_source, query_run) for query_run in self.run]
166  #return ["file {0}={1} run={2} ".format(query_by, query_source, query_run) for query_run in self.run]
167  else:
168  return ["file {0}={1} site=T2_CH_CERN".format(query_by, query_source)]
169 
170  def __str__(self):
171  if self.ib_block:
172  return "input from: {0} with run {1}#{2}".format(self.dataSet, self.ib_block, self.run)
173  return "input from: {0} with run {1}".format(self.dataSet, self.run)
174 
175 
176 # merge dictionaries, with prioty on the [0] index
177 def merge(dictlist,TELL=False):
178  import copy
179  last=len(dictlist)-1
180  if TELL: print last,dictlist
181  if last==0:
182  # ONLY ONE ITEM LEFT
183  return copy.copy(dictlist[0])
184  else:
185  reducedlist=dictlist[0:max(0,last-1)]
186  if TELL: print reducedlist
187  # make a copy of the last item
188  d=copy.copy(dictlist[last])
189  # update with the last but one item
190  d.update(dictlist[last-1])
191  # and recursively do the rest
192  reducedlist.append(d)
193  return merge(reducedlist,TELL)
194 
195 def remove(d,key,TELL=False):
196  import copy
197  e = copy.deepcopy(d)
198  if TELL: print "original dict, BEF: %s"%d
199  del e[key]
200  if TELL: print "copy-removed dict, AFT: %s"%e
201  return e
202 
203 
204 #### Standard release validation samples ####
205 
206 stCond={'--conditions':'auto:run1_mc'}
207 def Kby(N,s):
208  return {'--relval':'%s000,%s'%(N,s)}
209 def Mby(N,s):
210  return {'--relval':'%s000000,%s'%(N,s)}
211 
212 def changeRefRelease(steps,listOfPairs):
213  for s in steps:
214  if ('INPUT' in steps[s]):
215  oldD=steps[s]['INPUT'].dataSet
216  for (ref,newRef) in listOfPairs:
217  if ref in oldD:
218  steps[s]['INPUT'].dataSet=oldD.replace(ref,newRef)
219  if '--pileup_input' in steps[s]:
220  for (ref,newRef) in listOfPairs:
221  if ref in steps[s]['--pileup_input']:
222  steps[s]['--pileup_input']=steps[s]['--pileup_input'].replace(ref,newRef)
223 
224 def addForAll(steps,d):
225  for s in steps:
226  steps[s].update(d)
227 
228 
229 def genvalid(fragment,d,suffix='all',fi='',dataSet=''):
230  import copy
231  c=copy.copy(d)
232  if suffix:
233  c['-s']=c['-s'].replace('genvalid','genvalid_'+suffix)
234  if fi:
235  c['--filein']='lhe:%d'%(fi,)
236  if dataSet:
237  c['--filein']='das:%s'%(dataSet,)
238  c['cfg']=fragment
239  return c
240 
241 
Definition: merge.py:1
def selectedLS
Definition: MatrixUtil.py:62
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
def expandLsInterval
Definition: MatrixUtil.py:46
def changeRefRelease
Definition: MatrixUtil.py:212
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def genvalid
Definition: MatrixUtil.py:229
#define update(a, b)
def addForAll
Definition: MatrixUtil.py:224
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run