CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
dqm_interfaces.DQMcommunicator Class Reference
Inheritance diagram for dqm_interfaces.DQMcommunicator:

Public Member Functions

def __init__ (self, server, is_private=False, ident="DQMToJson/1.0 python/%d.%d.%d"%version_info[:3])
 
def cd (self, args)
 
def get_common_runs (self, dataset_string1, dataset_string2)
 
def get_data (self, full_url)
 
def get_dataset_runs (self, dataset_string)
 
def get_datasets_list (self, dataset_string="")
 
def get_RelVal_CMSSW_versions (self, query)
 
def get_root_objects (self, url="")
 
def get_root_objects_list (self, url="")
 
def get_root_objects_list_recursive (self, url="")
 
def get_root_objects_names_list_recursive (self, url="", present_url="")
 
def get_root_objects_recursive (self, url="", present_url="")
 
def get_runs_list (self, dataset_string)
 
def get_samples (self, samples_string="*")
 
def ls (self, url='', fetch_root=False)
 
def ls_url (self, url)
 
def open_url (self, url)
 

Public Attributes

 DQMpwd
 
 ident
 
 is_private
 
 oldDQMpwd
 
 opener
 
 prevDQMpwd
 
 server
 

Static Public Attributes

 base_dir
 

Detailed Description

Communicate with the DQM Document server

Definition at line 55 of file dqm_interfaces.py.

Constructor & Destructor Documentation

def dqm_interfaces.DQMcommunicator.__init__ (   self,
  server,
  is_private = False,
  ident = "DQMToJson/1.0 python/%d.%d.%d" % version_info[:3] 
)

Definition at line 66 of file dqm_interfaces.py.

66  ident="DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]):
67  self.ident = ident
68  self.server = server
69  self.is_private = is_private
70  self.DQMpwd=DQMcommunicator.base_dir
71  self.prevDQMpwd=self.DQMpwd
72  self.opener=None
73  if not self.is_private:
74  self.opener=build_opener(X509CertOpen())

Member Function Documentation

def dqm_interfaces.DQMcommunicator.cd (   self,
  args 
)

Definition at line 159 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.oldDQMpwd, and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DirWalkerFile.walk().

159  def cd(self, *args):
160  len_args=len(args)
161  full_url=""
162  if len_args!=1 and len_args!=3:
163  raise InvalidNumberOfArguments
164  if len_args==3:
165  dataset, run, folder = args
166  full_url='%s/data/json/archive/%s/%s/%s' % (self.server, dataset, run, folder)
167  if len_args==1:
168  folder=args[0]
169  if folder==self.DQMpwd:
170  full_url=self.DQMpwd
171  elif folder=="..":
172  full_url=self.DQMpwd[:self.DQMpwd.rfind("/")]
173  elif folder=="-":
174  full_url=self.oldDQMpwd
175  elif folder=="":
176  full_url=DQMcommunicator.base_dir
177  else:
178  full_url=self.DQMpwd+"/"+folder
179 
180  full_url=full_url.replace(' ','%20')
181  #print "cd: "+full_url
182 
183  self.oldDQMpwd=self.DQMpwd
184  self.DQMpwd=full_url
185  #print "In %s" %self.DQMpwd
186 
def dqm_interfaces.DQMcommunicator.get_common_runs (   self,
  dataset_string1,
  dataset_string2 
)

Definition at line 242 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_runs_list().

242  def get_common_runs(self,dataset_string1,dataset_string2):
243  set1=set(self.get_runs_list(dataset_string1))
244  set2=set(self.get_runs_list(dataset_string2))
245  set1.intersection_update(set2)
246  return list (set2)
247 
def get_common_runs(self, dataset_string1, dataset_string2)
def get_runs_list(self, dataset_string)
def dqm_interfaces.DQMcommunicator.get_data (   self,
  full_url 
)

Definition at line 92 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.open_url(), and readEcalDQMStatus.read.

Referenced by dqm_interfaces.DQMcommunicator.get_samples(), and dqm_interfaces.DQMcommunicator.ls_url().

92  def get_data(self, full_url):
93  #print "getting data from %s" %full_url
94  data = self.open_url(full_url).read()
95 
96  data = sub("-inf", '0', data)
97  data = sub("\s+inf", '0', data)
98  data = sub("\s+nan", '0', data)
99  data = sub('""(CMSSW.*?)""', '"\\1"', data)
100 
101  return data
102 
def get_data(self, full_url)
def dqm_interfaces.DQMcommunicator.get_dataset_runs (   self,
  dataset_string 
)

Definition at line 234 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_datasets_list(), and dqm_interfaces.DQMcommunicator.get_runs_list().

234  def get_dataset_runs(self,dataset_string):
235  dataset_runs={}
236  for dataset in self.get_datasets_list(dataset_string):
237  dataset_runs[dataset]=self.get_runs_list(dataset)
238  return dataset_runs
239 
def get_datasets_list(self, dataset_string="")
def get_dataset_runs(self, dataset_string)
def get_runs_list(self, dataset_string)
def dqm_interfaces.DQMcommunicator.get_datasets_list (   self,
  dataset_string = "" 
)

Definition at line 200 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_samples(), and genParticles_cff.map.

Referenced by dqm_interfaces.DQMcommunicator.get_dataset_runs(), and dqm_interfaces.DQMcommunicator.get_RelVal_CMSSW_versions().

200  def get_datasets_list(self, dataset_string=""):
201  samples_list=self.get_samples(dataset_string)
202  datasets_list=[]
203  for sample in samples_list:
204  temp_datasets_list = map(lambda item:item["dataset"] ,sample['items'])
205  for temp_dataset in temp_datasets_list:
206  if not temp_dataset in datasets_list:
207  datasets_list.append(temp_dataset)
208  return datasets_list
209 
def get_datasets_list(self, dataset_string="")
def get_samples(self, samples_string="*")
def dqm_interfaces.DQMcommunicator.get_RelVal_CMSSW_versions (   self,
  query 
)
Get the available cmssw versions for the relvals.

Definition at line 212 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_datasets_list(), and list().

212  def get_RelVal_CMSSW_versions(self,query):
213  """Get the available cmssw versions for the relvals.
214  """
215  relvals_list=self.get_datasets_list(query)
216  # The samples are of the form /RelValTHISISMYFAVOURITECHANNEL/CMSSW_VERSION/GEN-SIM-WHATEVER-RECO
217  cmssw_versions_with_duplicates=map (lambda x: x.split("/")[2],relvals_list)
218  return list(set(cmssw_versions_with_duplicates))
219 
def get_datasets_list(self, dataset_string="")
def get_RelVal_CMSSW_versions(self, query)
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def dqm_interfaces.DQMcommunicator.get_root_objects (   self,
  url = "" 
)

Definition at line 264 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, mps_monitormerge.items, ROOTData.literal2root(), WZInterestingEventSelector::event.ls, BeamSpotRcdReader::theBSfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, dqm_interfaces.DQMcommunicator.ls(), and dqm_interfaces.DQMcommunicator.server.

264  def get_root_objects(self, url=""):
265  if len(url)==0:
266  url=self.DQMpwd
267  else:
268  url=self.server+"/"+url
269  url = url.replace(" ","%20")
270  objects={}
271  for name,description in self.ls(url,True).items():
272  if "dir" not in description["type"] and "ROOT" in description["kind"]:
273  objects[name]=literal2root(description["obj_as_string"],description["type"])
274  return objects
275 
276  #-------------------------------------------------------------------------------
277 
def literal2root(literal, rootType, debug=False)
Definition: ROOTData.py:53
def ls(self, url='', fetch_root=False)
def get_root_objects(self, url="")
def dqm_interfaces.DQMcommunicator.get_root_objects_list (   self,
  url = "" 
)

Definition at line 250 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, mps_monitormerge.items, ROOTData.literal2root(), WZInterestingEventSelector::event.ls, BeamSpotRcdReader::theBSfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

250  def get_root_objects_list(self, url=""):
251  if len(url)==0:
252  url=self.DQMpwd
253  else:
254  url="/"+url
255  url = url.replace(" ","%20")
256  objects=[]
257  for name,description in self.ls(url,True).items():
258  if "dir" not in description["type"] and "ROOT" in description["kind"]:
259  objects.append(literal2root(description["obj_as_string"],description["type"]))
260  return objects
261 
def literal2root(literal, rootType, debug=False)
Definition: ROOTData.py:53
def ls(self, url='', fetch_root=False)
def get_root_objects_list(self, url="")
def dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive (   self,
  url = "" 
)

Definition at line 278 of file dqm_interfaces.py.

References TFileDirectory.cd(), TFileService.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), dqm::dqmstoreimpl::DQMStore::IBooker.cd(), dqm::dqmstoreimpl::DQMStore::IGetter.cd(), dqm::dqmstoreimpl::DQMStore.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), mps_monitormerge.items, ROOTData.literal2root(), WZInterestingEventSelector::event.ls, BeamSpotRcdReader::theBSfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive().

279  null_url = (len(url)==0)
280  if len(url)==0:
281  url=self.DQMpwd
282  else:
283  url="/"+url
284  url = url.replace(" ","%20")
285  if not null_url:
286  self.cd(url)
287  objects=[]
288  for name,description in self.ls("",True).items():
289  if "dir" in description["type"]:
290  objects+=self.get_root_objects_list_recursive(name)
291  self.cd("..")
292  elif "ROOT" in description["kind"]:
293  objects.append(literal2root(description["obj_as_string"],description["type"]))
294  if not null_url:
295  self.cd("..")
296  return objects
297 
298  #-------------------------------------------------------------------------------
299 
def literal2root(literal, rootType, debug=False)
Definition: ROOTData.py:53
def ls(self, url='', fetch_root=False)
def get_root_objects_list_recursive(self, url="")
def dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive (   self,
  url = "",
  present_url = "" 
)

Definition at line 300 of file dqm_interfaces.py.

References TFileDirectory.cd(), TFileService.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), dqm::dqmstoreimpl::DQMStore::IBooker.cd(), dqm::dqmstoreimpl::DQMStore::IGetter.cd(), dqm::dqmstoreimpl::DQMStore.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), mps_monitormerge.items, WZInterestingEventSelector::event.ls, BeamSpotRcdReader::theBSfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive().

300  def get_root_objects_names_list_recursive(self, url="",present_url=""):
301  null_url = (len(url)==0)
302  if (not null_url):
303  if len(present_url)==0:
304  present_url=url
305  else:
306  present_url+="_%s"%url
307  if len(url)==0:
308  url=self.DQMpwd
309  else:
310  url="/"+url
311  url = url.replace(" ","%20")
312  if not null_url:
313  self.cd(url)
314  objects_names=[]
315  for name,description in self.ls("",False).items():
316  if "dir" in description["type"]:
317  objects_names+=self.get_root_objects_names_list_recursive(name,present_url)
318  self.cd("..")
319  elif "ROOT" in description["kind"]:
320  objects_names.append("%s_%s"%(present_url,name))
321  if not null_url:
322  self.cd("..")
323  return objects_names
324 
325  #-------------------------------------------------------------------------------
326 
def ls(self, url='', fetch_root=False)
def get_root_objects_names_list_recursive(self, url="", present_url="")
def dqm_interfaces.DQMcommunicator.get_root_objects_recursive (   self,
  url = "",
  present_url = "" 
)

Definition at line 327 of file dqm_interfaces.py.

References TFileDirectory.cd(), TFileService.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), dqm::dqmstoreimpl::DQMStore::IBooker.cd(), dqm::dqmstoreimpl::DQMStore::IGetter.cd(), dqm::dqmstoreimpl::DQMStore.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), mps_monitormerge.items, ROOTData.literal2root(), WZInterestingEventSelector::event.ls, BeamSpotRcdReader::theBSfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_recursive().

327  def get_root_objects_recursive(self, url="",present_url=""):
328  null_url = (len(url)==0)
329  if (not null_url):
330  if len(present_url)==0:
331  present_url=url
332  else:
333  present_url+="_%s"%url
334  if len(url)==0:
335  url=self.DQMpwd
336  else:
337  url="/"+url
338  url = url.replace(" ","%20")
339  #if not null_url:
340  self.cd(url)
341  objects={}
342  for name,description in self.ls("",True).items():
343  if "dir" in description["type"]:
344  objects.update(self.get_root_objects_recursive(name,present_url))
345  self.cd("..")
346  elif "ROOT" in description["kind"]:
347  objects["%s_%s"%(present_url,name)]=literal2root(description["obj_as_string"],description["type"])
348  #if not null_url:
349  self.cd("..")
350  return objects
351 
352 #-------------------------------------------------------------------------------
353 
def literal2root(literal, rootType, debug=False)
Definition: ROOTData.py:53
def ls(self, url='', fetch_root=False)
def get_root_objects_recursive(self, url="", present_url="")
def dqm_interfaces.DQMcommunicator.get_runs_list (   self,
  dataset_string 
)

Definition at line 222 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_samples(), and genParticles_cff.map.

Referenced by dqm_interfaces.DQMcommunicator.get_common_runs(), and dqm_interfaces.DQMcommunicator.get_dataset_runs().

222  def get_runs_list(self, dataset_string):
223  slash="/"
224  while(dataset_string.endswith(slash) or dataset_string.beginswith(slash)):
225  dataset_string=dataset_string.strip("/")
226  samples_list=self.get_samples(dataset_string)
227  runlist=[]
228  # Get all the runs in all the items which are in every sample
229  map( lambda sample: map (lambda item: runlist.append(item['run']), sample['items']), samples_list)
230  return runlist
231 
def get_runs_list(self, dataset_string)
def get_samples(self, samples_string="*")
def dqm_interfaces.DQMcommunicator.get_samples (   self,
  samples_string = "*" 
)
A sample contains, among the other things, a data type, a dataset name 
and a run.

Definition at line 189 of file dqm_interfaces.py.

References data_sources.data_source.get_data(), dqm_interfaces.DQMcommunicator.get_data(), and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.get_datasets_list(), and dqm_interfaces.DQMcommunicator.get_runs_list().

189  def get_samples(self, samples_string="*"):
190  """
191  A sample contains, among the other things, a data type, a dataset name
192  and a run.
193  """
194  full_url='%s/data/json/samples?match=%s' % (self.server, samples_string)
195  samples_dict=eval(self.get_data(full_url))
196  return samples_dict["samples"]
197 
def get_data(self, full_url)
def get_samples(self, samples_string="*")
def dqm_interfaces.DQMcommunicator.ls (   self,
  url = '',
  fetch_root = False 
)

Definition at line 145 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, join(), and dqm_interfaces.DQMcommunicator.ls_url().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects(), dqm_interfaces.DQMcommunicator.get_root_objects_list(), dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DirWalkerFile.walk().

145  def ls(self, url='', fetch_root=False):
146  if len(url)==0:
147  url=join(self.DQMpwd,url)
148 
149  form_folder={}
150 
151  if fetch_root:
152  url='%s?rootcontent=1'%url
153  form_folder=self.ls_url(url)
154 
155  return form_folder
156 
def ls(self, url='', fetch_root=False)
static std::string join(char **cmd)
Definition: RemoteFile.cc:17
def dqm_interfaces.DQMcommunicator.ls_url (   self,
  url 
)

Definition at line 105 of file dqm_interfaces.py.

References data_sources.data_source.get_data(), dqm_interfaces.DQMcommunicator.get_data(), edm.print(), FastTimerService_cff.range, and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.ls().

105  def ls_url(self, url):
106  url=url.replace(" ","%20")
107  url=self.server+url
108  #print "listing "+url
109  form_folder={}
110  raw_folder=None
111  try:
112  raw_folder=eval(self.get_data(url))
113  except:
114  print("Retrying..")
115  for ntrials in range(5):
116  try:
117  if ntrials!=0:
118  sleep(2)
119  #raw_folder=loads(self.get_data(url))
120  raw_folder=eval(self.get_data(url))
121  break
122  except:
123  print("Could not fetch %s. Retrying" %url)
124 
125  #raw_folder=loads(self.get_data(url))
126  for content_dict in raw_folder["contents"]:
127  if "subdir" in content_dict:
128  form_folder[content_dict["subdir"]]={"type":'dir'}
129  elif "obj" in content_dict:
130  properties=content_dict["properties"]
131  obj_name=content_dict["obj"]
132  obj_type=properties["type"]
133  obj_kind=properties["kind"]
134  obj_as_string=''
135  if "rootobj" in content_dict:
136  obj_as_string=content_dict["rootobj"]
137  form_folder[obj_name]={'type':obj_type,'obj_as_string':obj_as_string,"kind":obj_kind}
138  #for k,v in form_folder.items():
139  #print "* %s --> %s" %(k,v["type"])
140 
141  return form_folder
142 
def get_data(self, full_url)
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
def dqm_interfaces.DQMcommunicator.open_url (   self,
  url 
)

Definition at line 77 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.ident, and dqm_interfaces.DQMcommunicator.is_private.

Referenced by dqm_interfaces.DQMcommunicator.get_data().

77  def open_url(self,url):
78  url=url.replace(' ','%20')
79  datareq = Request(url)
80  datareq.add_header('User-agent', self.ident)
81  url_obj=0
82  if not self.is_private:
83  url_obj=self.opener.open(datareq)
84  #url_obj=build_opener(X509CertOpen()).open(datareq)
85  else:
86  url_obj=urlopen(datareq)
87 
88  return url_obj
89 

Member Data Documentation

dqm_interfaces.DQMcommunicator.base_dir
static

Definition at line 61 of file dqm_interfaces.py.

dqm_interfaces.DQMcommunicator.DQMpwd
dqm_interfaces.DQMcommunicator.ident

Definition at line 67 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.open_url().

dqm_interfaces.DQMcommunicator.is_private

Definition at line 69 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.open_url().

dqm_interfaces.DQMcommunicator.oldDQMpwd

Definition at line 183 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.cd().

dqm_interfaces.DQMcommunicator.opener

Definition at line 72 of file dqm_interfaces.py.

dqm_interfaces.DQMcommunicator.prevDQMpwd

Definition at line 71 of file dqm_interfaces.py.

dqm_interfaces.DQMcommunicator.server