CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
dqm_interfaces.DQMcommunicator Class Reference
Inheritance diagram for dqm_interfaces.DQMcommunicator:

Public Member Functions

def __init__
 
def cd
 
def get_common_runs
 
def get_data
 
def get_dataset_runs
 
def get_datasets_list
 
def get_RelVal_CMSSW_versions
 
def get_root_objects
 
def get_root_objects_list
 
def get_root_objects_list_recursive
 
def get_root_objects_names_list_recursive
 
def get_root_objects_recursive
 
def get_runs_list
 
def get_samples
 
def ls
 
def ls_url
 
def open_url
 

Public Attributes

 DQMpwd
 
 ident
 
 is_private
 
 oldDQMpwd
 
 opener
 
 prevDQMpwd
 
 server
 

Static Public Attributes

string base_dir = '/data/json/archive/'
 

Detailed Description

Communicate with the DQM Document server

Definition at line 55 of file dqm_interfaces.py.

Constructor & Destructor Documentation

def dqm_interfaces.DQMcommunicator.__init__ (   self,
  server,
  is_private = False,
  ident = "DQMToJson/1.0 python/%d.%d.%d" % version_info[:3] 
)

Definition at line 66 of file dqm_interfaces.py.

66 
67  ident="DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]):
68  self.ident = ident
69  self.server = server
70  self.is_private = is_private
71  self.DQMpwd=DQMcommunicator.base_dir
72  self.prevDQMpwd=self.DQMpwd
73  self.opener=None
74  if not self.is_private:
self.opener=build_opener(X509CertOpen())

Member Function Documentation

def dqm_interfaces.DQMcommunicator.cd (   self,
  args 
)

Definition at line 159 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.oldDQMpwd, and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DirWalkerFile.walk().

160  def cd(self, *args):
161  len_args=len(args)
162  full_url=""
163  if len_args!=1 and len_args!=3:
164  raise(InvalidNumberOfArguments("3 or 1 args expected!"))
165  if len_args==3:
166  dataset, run, folder = args
167  full_url='%s/data/json/archive/%s/%s/%s' % (self.server, dataset, run, folder)
168  if len_args==1:
169  folder=args[0]
170  if folder==self.DQMpwd:
171  full_url=self.DQMpwd
172  elif folder=="..":
173  full_url=self.DQMpwd[:self.DQMpwd.rfind("/")]
174  elif folder=="-":
175  full_url=self.oldDQMpwd
176  elif folder=="":
177  full_url=DQMcommunicator.base_dir
178  else:
179  full_url=self.DQMpwd+"/"+folder
180 
181  full_url=full_url.replace(' ','%20')
182  #print "cd: "+full_url
183 
184  self.oldDQMpwd=self.DQMpwd
185  self.DQMpwd=full_url
186  #print "In %s" %self.DQMpwd
def dqm_interfaces.DQMcommunicator.get_common_runs (   self,
  dataset_string1,
  dataset_string2 
)

Definition at line 242 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_runs_list(), and runtimedef.set().

243  def get_common_runs(self,dataset_string1,dataset_string2):
244  set1=set(self.get_runs_list(dataset_string1))
245  set2=set(self.get_runs_list(dataset_string2))
246  set1.intersection_update(set2)
247  return list (set2)
void set(const std::string &name, int value)
set the flag, with a run-time name
def dqm_interfaces.DQMcommunicator.get_data (   self,
  full_url 
)

Definition at line 92 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.open_url(), and SiPixelLorentzAngle_cfi.read.

Referenced by dqm_interfaces.DQMcommunicator.get_samples(), and dqm_interfaces.DQMcommunicator.ls_url().

92 
93  def get_data(self, full_url):
94  #print "getting data from %s" %full_url
95  data = self.open_url(full_url).read()
96 
97  data = sub("-inf", '0', data)
98  data = sub("\s+inf", '0', data)
99  data = sub("\s+nan", '0', data)
100  data = sub('""(CMSSW.*?)""', '"\\1"', data)
101 
102  return data
def dqm_interfaces.DQMcommunicator.get_dataset_runs (   self,
  dataset_string 
)

Definition at line 234 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_datasets_list(), and dqm_interfaces.DQMcommunicator.get_runs_list().

235  def get_dataset_runs(self,dataset_string):
236  dataset_runs={}
237  for dataset in self.get_datasets_list(dataset_string):
238  dataset_runs[dataset]=self.get_runs_list(dataset)
239  return dataset_runs
def dqm_interfaces.DQMcommunicator.get_datasets_list (   self,
  dataset_string = "" 
)

Definition at line 200 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_samples(), and python.multivaluedict.map().

Referenced by dqm_interfaces.DQMcommunicator.get_dataset_runs(), and dqm_interfaces.DQMcommunicator.get_RelVal_CMSSW_versions().

201  def get_datasets_list(self, dataset_string=""):
202  samples_list=self.get_samples(dataset_string)
203  datasets_list=[]
204  for sample in samples_list:
205  temp_datasets_list = map(lambda item:item["dataset"] ,sample['items'])
206  for temp_dataset in temp_datasets_list:
207  if not temp_dataset in datasets_list:
208  datasets_list.append(temp_dataset)
209  return datasets_list
def dqm_interfaces.DQMcommunicator.get_RelVal_CMSSW_versions (   self,
  query 
)
Get the available cmssw versions for the relvals.

Definition at line 212 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_datasets_list(), list(), and runtimedef.set().

213  def get_RelVal_CMSSW_versions(self,query):
214  """Get the available cmssw versions for the relvals.
215  """
216  relvals_list=self.get_datasets_list(query)
217  # The samples are of the form /RelValTHISISMYFAVOURITECHANNEL/CMSSW_VERSION/GEN-SIM-WHATEVER-RECO
218  cmssw_versions_with_duplicates=map (lambda x: x.split("/")[2],relvals_list)
219  return list(set(cmssw_versions_with_duplicates))
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
void set(const std::string &name, int value)
set the flag, with a run-time name
def dqm_interfaces.DQMcommunicator.get_root_objects (   self,
  url = "" 
)

Definition at line 264 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, ROOTData.literal2root(), evf::lsTriplet.ls, ProcLinear.ls, WZInterestingEventSelector::event.ls, evf::prg.ls, ProcMatrix.ls, EcalDeadCellDeltaRFilter.ls, EcalDeadCellTriggerPrimitiveFilter.ls, dqm_interfaces.DQMcommunicator.ls(), and dqm_interfaces.DQMcommunicator.server.

265  def get_root_objects(self, url=""):
266  if len(url)==0:
267  url=self.DQMpwd
268  else:
269  url=self.server+"/"+url
270  url = url.replace(" ","%20")
271  objects={}
272  for name,description in self.ls(url,True).items():
273  if "dir" not in description["type"] and "ROOT" in description["kind"]:
274  objects[name]=literal2root(description["obj_as_string"],description["type"])
275  return objects
def literal2root
Definition: ROOTData.py:24
def dqm_interfaces.DQMcommunicator.get_root_objects_list (   self,
  url = "" 
)

Definition at line 250 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, ROOTData.literal2root(), evf::lsTriplet.ls, ProcLinear.ls, WZInterestingEventSelector::event.ls, evf::prg.ls, ProcMatrix.ls, EcalDeadCellDeltaRFilter.ls, EcalDeadCellTriggerPrimitiveFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

251  def get_root_objects_list(self, url=""):
252  if len(url)==0:
253  url=self.DQMpwd
254  else:
255  url="/"+url
256  url = url.replace(" ","%20")
257  objects=[]
258  for name,description in self.ls(url,True).items():
259  if "dir" not in description["type"] and "ROOT" in description["kind"]:
260  objects.append(literal2root(description["obj_as_string"],description["type"]))
261  return objects
def literal2root
Definition: ROOTData.py:24
def dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive (   self,
  url = "" 
)

Definition at line 278 of file dqm_interfaces.py.

References TFileDirectory.cd(), ResidualFitter.cd(), DQMStore.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), ROOTData.literal2root(), evf::lsTriplet.ls, ProcLinear.ls, WZInterestingEventSelector::event.ls, evf::prg.ls, ProcMatrix.ls, EcalDeadCellDeltaRFilter.ls, EcalDeadCellTriggerPrimitiveFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive().

279  def get_root_objects_list_recursive(self, url=""):
280  null_url = (len(url)==0)
281  if len(url)==0:
282  url=self.DQMpwd
283  else:
284  url="/"+url
285  url = url.replace(" ","%20")
286  if not null_url:
287  self.cd(url)
288  objects=[]
289  for name,description in self.ls("",True).items():
290  if "dir" in description["type"]:
291  objects+=self.get_root_objects_list_recursive(name)
292  self.cd("..")
293  elif "ROOT" in description["kind"]:
294  objects.append(literal2root(description["obj_as_string"],description["type"]))
295  if not null_url:
296  self.cd("..")
297  return objects
def literal2root
Definition: ROOTData.py:24
def dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive (   self,
  url = "",
  present_url = "" 
)

Definition at line 300 of file dqm_interfaces.py.

References TFileDirectory.cd(), ResidualFitter.cd(), DQMStore.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), evf::lsTriplet.ls, ProcLinear.ls, WZInterestingEventSelector::event.ls, evf::prg.ls, ProcMatrix.ls, EcalDeadCellDeltaRFilter.ls, EcalDeadCellTriggerPrimitiveFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive().

301  def get_root_objects_names_list_recursive(self, url="",present_url=""):
302  null_url = (len(url)==0)
303  if (not null_url):
304  if len(present_url)==0:
305  present_url=url
306  else:
307  present_url+="_%s"%url
308  if len(url)==0:
309  url=self.DQMpwd
310  else:
311  url="/"+url
312  url = url.replace(" ","%20")
313  if not null_url:
314  self.cd(url)
315  objects_names=[]
316  for name,description in self.ls("",False).items():
317  if "dir" in description["type"]:
318  objects_names+=self.get_root_objects_names_list_recursive(name,present_url)
319  self.cd("..")
320  elif "ROOT" in description["kind"]:
321  objects_names.append("%s_%s"%(present_url,name))
322  if not null_url:
323  self.cd("..")
324  return objects_names
def dqm_interfaces.DQMcommunicator.get_root_objects_recursive (   self,
  url = "",
  present_url = "" 
)

Definition at line 327 of file dqm_interfaces.py.

References TFileDirectory.cd(), ResidualFitter.cd(), DQMStore.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), ROOTData.literal2root(), evf::lsTriplet.ls, ProcLinear.ls, WZInterestingEventSelector::event.ls, evf::prg.ls, ProcMatrix.ls, EcalDeadCellDeltaRFilter.ls, EcalDeadCellTriggerPrimitiveFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_recursive().

328  def get_root_objects_recursive(self, url="",present_url=""):
329  null_url = (len(url)==0)
330  if (not null_url):
331  if len(present_url)==0:
332  present_url=url
333  else:
334  present_url+="_%s"%url
335  if len(url)==0:
336  url=self.DQMpwd
337  else:
338  url="/"+url
339  url = url.replace(" ","%20")
340  #if not null_url:
341  self.cd(url)
342  objects={}
343  for name,description in self.ls("",True).items():
344  if "dir" in description["type"]:
345  objects.update(self.get_root_objects_recursive(name,present_url))
346  self.cd("..")
347  elif "ROOT" in description["kind"]:
348  objects["%s_%s"%(present_url,name)]=literal2root(description["obj_as_string"],description["type"])
349  #if not null_url:
350  self.cd("..")
351  return objects
352 
353 #-------------------------------------------------------------------------------
def literal2root
Definition: ROOTData.py:24
def dqm_interfaces.DQMcommunicator.get_runs_list (   self,
  dataset_string 
)

Definition at line 222 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_samples(), and python.multivaluedict.map().

Referenced by dqm_interfaces.DQMcommunicator.get_common_runs(), and dqm_interfaces.DQMcommunicator.get_dataset_runs().

223  def get_runs_list(self, dataset_string):
224  slash="/"
225  while(dataset_string.endswith(slash) or dataset_string.beginswith(slash)):
226  dataset_string=dataset_string.strip("/")
227  samples_list=self.get_samples(dataset_string)
228  runlist=[]
229  # Get all the runs in all the items which are in every sample
230  map( lambda sample: map (lambda item: runlist.append(item['run']), sample['items']), samples_list)
231  return runlist
def dqm_interfaces.DQMcommunicator.get_samples (   self,
  samples_string = "*" 
)
A sample contains, among the other things, a data type, a dataset name 
and a run.

Definition at line 189 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_data(), and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.get_datasets_list(), and dqm_interfaces.DQMcommunicator.get_runs_list().

190  def get_samples(self, samples_string="*"):
191  """
192  A sample contains, among the other things, a data type, a dataset name
193  and a run.
194  """
195  full_url='%s/data/json/samples?match=%s' % (self.server, samples_string)
196  samples_dict=eval(self.get_data(full_url))
197  return samples_dict["samples"]
def dqm_interfaces.DQMcommunicator.ls (   self,
  url = '',
  fetch_root = False 
)

Definition at line 145 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, join(), and dqm_interfaces.DQMcommunicator.ls_url().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects(), dqm_interfaces.DQMcommunicator.get_root_objects_list(), dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DirWalkerFile.walk().

146  def ls(self, url='', fetch_root=False):
147  if len(url)==0:
148  url=join(self.DQMpwd,url)
149 
150  form_folder={}
151 
152  if fetch_root:
153  url='%s?rootcontent=1'%url
154  form_folder=self.ls_url(url)
155 
156  return form_folder
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def dqm_interfaces.DQMcommunicator.ls_url (   self,
  url 
)

Definition at line 105 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_data(), dqm_interfaces.DQMcommunicator.server, and stor::utils.sleep().

Referenced by dqm_interfaces.DQMcommunicator.ls().

106  def ls_url(self, url):
107  url=url.replace(" ","%20")
108  url=self.server+url
109  #print "listing "+url
110  form_folder={}
111  raw_folder=None
112  try:
113  raw_folder=eval(self.get_data(url))
114  except:
115  print "Retrying.."
116  for ntrials in xrange(5):
117  try:
118  if ntrials!=0:
119  sleep(2)
120  #raw_folder=loads(self.get_data(url))
121  raw_folder=eval(self.get_data(url))
122  break
123  except:
124  print "Could not fetch %s. Retrying" %url
125 
126  #raw_folder=loads(self.get_data(url))
127  for content_dict in raw_folder["contents"]:
128  if content_dict.has_key("subdir"):
129  form_folder[content_dict["subdir"]]={"type":'dir'}
130  elif content_dict.has_key("obj"):
131  properties=content_dict["properties"]
132  obj_name=content_dict["obj"]
133  obj_type=properties["type"]
134  obj_kind=properties["kind"]
135  obj_as_string=''
136  if content_dict.has_key("rootobj"):
137  obj_as_string=content_dict["rootobj"]
138  form_folder[obj_name]={'type':obj_type,'obj_as_string':obj_as_string,"kind":obj_kind}
139  #for k,v in form_folder.items():
140  #print "* %s --> %s" %(k,v["type"])
141 
142  return form_folder
void sleep(Duration_t)
Definition: Utils.h:163
def dqm_interfaces.DQMcommunicator.open_url (   self,
  url 
)

Definition at line 77 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.ident, and dqm_interfaces.DQMcommunicator.is_private.

Referenced by dqm_interfaces.DQMcommunicator.get_data().

77 
78  def open_url(self,url):
79  url=url.replace(' ','%20')
80  datareq = Request(url)
81  datareq.add_header('User-agent', self.ident)
82  url_obj=0
83  if not self.is_private:
84  url_obj=self.opener.open(datareq)
85  #url_obj=build_opener(X509CertOpen()).open(datareq)
86  else:
87  url_obj=urlopen(datareq)
88 
89  return url_obj

Member Data Documentation

string dqm_interfaces.DQMcommunicator.base_dir = '/data/json/archive/'
static

Definition at line 61 of file dqm_interfaces.py.

dqm_interfaces.DQMcommunicator.DQMpwd

Definition at line 70 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.cd(), dqm_interfaces.DQMcommunicator.get_root_objects(), dqm_interfaces.DQMcommunicator.get_root_objects_list(), dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DQMcommunicator.ls().

dqm_interfaces.DQMcommunicator.ident

Definition at line 67 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.open_url().

dqm_interfaces.DQMcommunicator.is_private

Definition at line 69 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.open_url().

dqm_interfaces.DQMcommunicator.oldDQMpwd

Definition at line 183 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.cd().

dqm_interfaces.DQMcommunicator.opener

Definition at line 72 of file dqm_interfaces.py.

dqm_interfaces.DQMcommunicator.prevDQMpwd

Definition at line 71 of file dqm_interfaces.py.

dqm_interfaces.DQMcommunicator.server

Definition at line 68 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.cd(), dqm_interfaces.DQMcommunicator.get_root_objects(), dqm_interfaces.DQMcommunicator.get_samples(), and dqm_interfaces.DQMcommunicator.ls_url().