Inheritance diagram for dqm_interfaces.DQMcommunicator:

Public Member Functions
def	__init__ (self, server, is_private=False, ident="DQMToJson/1.0 python/%d.%d.%d"%version_info[:3])

def	cd (self, args)

def	get_common_runs (self, dataset_string1, dataset_string2)

def	get_data (self, full_url)

def	get_dataset_runs (self, dataset_string)

def	get_datasets_list (self, dataset_string="")

def	get_RelVal_CMSSW_versions (self, query)

def	get_root_objects (self, url="")

def	get_root_objects_list (self, url="")

def	get_root_objects_list_recursive (self, url="")

def	get_root_objects_names_list_recursive (self, url="", present_url="")

def	get_root_objects_recursive (self, url="", present_url="")

def	get_runs_list (self, dataset_string)

def	get_samples (self, samples_string="*")

def	ls (self, url='', fetch_root=False)

def	ls_url (self, url)

def	open_url (self, url)

Public Attributes
	DQMpwd

	ident

	is_private

	oldDQMpwd

	opener

	prevDQMpwd

	server

Static Public Attributes
	base_dir

Detailed Description

Communicate with the DQM Document server

Definition at line 52 of file dqm_interfaces.py.

Constructor & Destructor Documentation

def dqm_interfaces.DQMcommunicator.__init__	(	self,
		server,
		is_private = `False`,
		ident = `"DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]`
	)

Definition at line 63 of file dqm_interfaces.py.

                ident="DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]):
     self.ident = ident
     self.server = server
     self.is_private = is_private
     self.DQMpwd=DQMcommunicator.base_dir
     self.prevDQMpwd=self.DQMpwd
     self.opener=None
     if not self.is_private:
       self.opener=build_opener(X509CertOpen())

Member Function Documentation

def dqm_interfaces.DQMcommunicator.cd	(	self,
		args
	)

Definition at line 156 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.oldDQMpwd, and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DirWalkerFile.walk().

   def cd(self, *args):
     len_args=len(args)
     full_url=""
     if len_args!=1 and len_args!=3:
       raise InvalidNumberOfArguments
     if len_args==3:
       dataset, run, folder = args    
       full_url='%s/data/json/archive/%s/%s/%s' % (self.server, dataset, run, folder)
     if len_args==1:
       folder=args[0]
       if folder==self.DQMpwd:
         full_url=self.DQMpwd
       elif folder=="..":
         full_url=self.DQMpwd[:self.DQMpwd.rfind("/")]
       elif folder=="-":
         full_url=self.oldDQMpwd
       elif folder=="":
         full_url=DQMcommunicator.base_dir
       else:
         full_url=self.DQMpwd+"/"+folder
         
     full_url=full_url.replace(' ','%20')
     #print "cd: "+full_url
     
     self.oldDQMpwd=self.DQMpwd
     self.DQMpwd=full_url   
     #print "In %s" %self.DQMpwd
   

def dqm_interfaces.DQMcommunicator.get_common_runs	(	self,
		dataset_string1,
		dataset_string2
	)

Definition at line 239 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_runs_list().

   def get_common_runs(self,dataset_string1,dataset_string2):
     set1=set(self.get_runs_list(dataset_string1))
     set2=set(self.get_runs_list(dataset_string2))
     set1.intersection_update(set2)
     return list (set2)
 

def dqm_interfaces.DQMcommunicator.get_data	(	self,
		full_url
	)

Definition at line 89 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.open_url().

Referenced by dqm_interfaces.DQMcommunicator.get_samples(), and dqm_interfaces.DQMcommunicator.ls_url().

   def get_data(self, full_url):
     #print "getting data from %s" %full_url
     data = self.open_url(full_url).read()
     
     data = sub("-inf", '0', data)
     data = sub("\s+inf", '0', data)
     data = sub("\s+nan", '0', data)
     data = sub('""(CMSSW.*?)""', '"\\1"', data)
    
     return data
   

def dqm_interfaces.DQMcommunicator.get_dataset_runs	(	self,
		dataset_string
	)

Definition at line 231 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_datasets_list(), and dqm_interfaces.DQMcommunicator.get_runs_list().

   def get_dataset_runs(self,dataset_string):
     dataset_runs={}
     for dataset in self.get_datasets_list(dataset_string):
       dataset_runs[dataset]=self.get_runs_list(dataset)
     return dataset_runs
 

def dqm_interfaces.DQMcommunicator.get_datasets_list	(	self,
		dataset_string = `""`
	)

Definition at line 197 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_samples(), and genParticles_cff.map.

Referenced by dqm_interfaces.DQMcommunicator.get_dataset_runs(), and dqm_interfaces.DQMcommunicator.get_RelVal_CMSSW_versions().

   def get_datasets_list(self, dataset_string=""):
     samples_list=self.get_samples(dataset_string)    
     datasets_list=[]
     for sample in samples_list:
       temp_datasets_list =  map(lambda item:item["dataset"] ,sample['items'])
       for temp_dataset in temp_datasets_list:
         if not temp_dataset in datasets_list:
           datasets_list.append(temp_dataset)
     return datasets_list
     

def dqm_interfaces.DQMcommunicator.get_RelVal_CMSSW_versions	(	self,
		query
	)

Get the available cmssw versions for the relvals.

Definition at line 209 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_datasets_list(), and list().

   def get_RelVal_CMSSW_versions(self,query):
     """Get the available cmssw versions for the relvals.
     """
     relvals_list=self.get_datasets_list(query)
     # The samples are of the form /RelValTHISISMYFAVOURITECHANNEL/CMSSW_VERSION/GEN-SIM-WHATEVER-RECO
     cmssw_versions_with_duplicates=map (lambda x: x.split("/")[2],relvals_list)
     return list(set(cmssw_versions_with_duplicates))
     

def dqm_interfaces.DQMcommunicator.get_root_objects	(	self,
		url = `""`
	)

Definition at line 261 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, mps_monitormerge.items, ROOTData.literal2root(), ProcLinear.ls, WZInterestingEventSelector::event.ls, ProcMatrix.ls, BeamSpotRcdReader::theBSfromDB.ls, create_public_lumi_plots.LumiDataPoint.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, dqm_interfaces.DQMcommunicator.ls(), and dqm_interfaces.DQMcommunicator.server.

   def get_root_objects(self, url=""):
     if len(url)==0:
       url=self.DQMpwd
     else:
       url=self.server+"/"+url    
     url = url.replace(" ","%20")
     objects={}
     for name,description in self.ls(url,True).items():     
       if "dir" not in description["type"] and "ROOT" in description["kind"]:
         objects[name]=literal2root(description["obj_as_string"],description["type"])
     return objects
 

def dqm_interfaces.DQMcommunicator.get_root_objects_list	(	self,
		url = `""`
	)

Definition at line 247 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, mps_monitormerge.items, ROOTData.literal2root(), ProcLinear.ls, WZInterestingEventSelector::event.ls, ProcMatrix.ls, BeamSpotRcdReader::theBSfromDB.ls, create_public_lumi_plots.LumiDataPoint.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

   def get_root_objects_list(self, url=""):
     if len(url)==0:
       url=self.DQMpwd
     else:
       url="/"+url    
     url = url.replace(" ","%20")
     objects=[]
     for name,description in self.ls(url,True).items():     
       if "dir" not in description["type"]  and "ROOT" in description["kind"]:
         objects.append(literal2root(description["obj_as_string"],description["type"]))
     return objects
 

def dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive	(	self,
		url = `""`
	)

Definition at line 275 of file dqm_interfaces.py.

References TFileDirectory.cd(), TFileService.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), DQMStore::IBooker.cd(), DQMStore::IGetter.cd(), python.rootplot.utilities.RootFile.cd(), DQMStore.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), mps_monitormerge.items, ROOTData.literal2root(), ProcLinear.ls, WZInterestingEventSelector::event.ls, ProcMatrix.ls, BeamSpotRcdReader::theBSfromDB.ls, create_public_lumi_plots.LumiDataPoint.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive().

   def get_root_objects_list_recursive(self, url=""):
     null_url = (len(url)==0)    
     if len(url)==0:
       url=self.DQMpwd
     else:
       url="/"+url    
     url = url.replace(" ","%20")      
     if not null_url: 
       self.cd(url)
     objects=[]
     for name,description in self.ls("",True).items():     
       if "dir" in description["type"]:
         objects+=self.get_root_objects_list_recursive(name)
         self.cd("..")
       elif  "ROOT" in description["kind"]:
         objects.append(literal2root(description["obj_as_string"],description["type"]))
     if not null_url: 
       self.cd("..")
     return objects
     

def dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive	(	self,
		url = `""`,
		present_url = `""`
	)

Definition at line 297 of file dqm_interfaces.py.

References TFileDirectory.cd(), TFileService.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), DQMStore::IBooker.cd(), DQMStore::IGetter.cd(), python.rootplot.utilities.RootFile.cd(), DQMStore.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), mps_monitormerge.items, ProcLinear.ls, WZInterestingEventSelector::event.ls, ProcMatrix.ls, BeamSpotRcdReader::theBSfromDB.ls, create_public_lumi_plots.LumiDataPoint.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive().

   def get_root_objects_names_list_recursive(self, url="",present_url=""):
     null_url = (len(url)==0)
     if (not null_url):
       if len(present_url)==0:
         present_url=url
       else:
         present_url+="_%s"%url
     if len(url)==0:
       url=self.DQMpwd
     else:
       url="/"+url    
     url = url.replace(" ","%20")
     if not null_url:
       self.cd(url)
     objects_names=[]
     for name,description in self.ls("",False).items():     
       if "dir" in description["type"]:        
         objects_names+=self.get_root_objects_names_list_recursive(name,present_url)
         self.cd("..")
       elif  "ROOT" in description["kind"]:
         objects_names.append("%s_%s"%(present_url,name))
     if not null_url: 
       self.cd("..")
     return objects_names
     

def dqm_interfaces.DQMcommunicator.get_root_objects_recursive	(	self,
		url = `""`,
		present_url = `""`
	)

Definition at line 324 of file dqm_interfaces.py.

References TFileDirectory.cd(), TFileService.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), DQMStore::IBooker.cd(), DQMStore::IGetter.cd(), python.rootplot.utilities.RootFile.cd(), DQMStore.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), mps_monitormerge.items, ROOTData.literal2root(), ProcLinear.ls, WZInterestingEventSelector::event.ls, ProcMatrix.ls, BeamSpotRcdReader::theBSfromDB.ls, create_public_lumi_plots.LumiDataPoint.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_recursive().

   def get_root_objects_recursive(self, url="",present_url=""):
     null_url = (len(url)==0)
     if (not null_url):
       if len(present_url)==0:
         present_url=url
       else:
         present_url+="_%s"%url
     if len(url)==0:
       url=self.DQMpwd
     else:
       url="/"+url    
     url = url.replace(" ","%20")
     #if not null_url:
     self.cd(url)
     objects={}
     for name,description in self.ls("",True).items():     
       if "dir" in description["type"]:
         objects.update(self.get_root_objects_recursive(name,present_url))
         self.cd("..")
       elif  "ROOT" in description["kind"]:
         objects["%s_%s"%(present_url,name)]=literal2root(description["obj_as_string"],description["type"])
     #if not null_url:
     self.cd("..")
     return objects
 
 #-------------------------------------------------------------------------------
 

def dqm_interfaces.DQMcommunicator.get_runs_list	(	self,
		dataset_string
	)

Definition at line 219 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_samples(), and genParticles_cff.map.

Referenced by dqm_interfaces.DQMcommunicator.get_common_runs(), and dqm_interfaces.DQMcommunicator.get_dataset_runs().

   def get_runs_list(self, dataset_string):
     slash="/"
     while(dataset_string.endswith(slash) or dataset_string.beginswith(slash)):
       dataset_string=dataset_string.strip("/")
     samples_list=self.get_samples(dataset_string)
     runlist=[]
     # Get all the runs in all the items which are in every sample
     map( lambda sample: map (lambda item: runlist.append(item['run']), sample['items']), samples_list)
     return runlist
   

def dqm_interfaces.DQMcommunicator.get_samples	(	self,
		samples_string = `"*"`
	)

A sample contains, among the other things, a data type, a dataset name 
and a run.

Definition at line 186 of file dqm_interfaces.py.

References data_sources.data_source.get_data(), dqm_interfaces.DQMcommunicator.get_data(), and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.get_datasets_list(), and dqm_interfaces.DQMcommunicator.get_runs_list().

   def get_samples(self, samples_string="*"):
     """
     A sample contains, among the other things, a data type, a dataset name 
     and a run.
     """
     full_url='%s/data/json/samples?match=%s' % (self.server, samples_string)
     samples_dict=eval(self.get_data(full_url))
     return samples_dict["samples"]
 

def dqm_interfaces.DQMcommunicator.ls	(	self,
		url = `''`,
		fetch_root = `False`
	)

Definition at line 142 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, join(), and dqm_interfaces.DQMcommunicator.ls_url().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects(), dqm_interfaces.DQMcommunicator.get_root_objects_list(), dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DirWalkerFile.walk().

   def ls(self, url='', fetch_root=False):
     if len(url)==0:
       url=join(self.DQMpwd,url)
     
     form_folder={}   
     
     if fetch_root:
       url='%s?rootcontent=1'%url
     form_folder=self.ls_url(url)
 
     return form_folder
     

def dqm_interfaces.DQMcommunicator.ls_url	(	self,
		url
	)

Definition at line 102 of file dqm_interfaces.py.

References data_sources.data_source.get_data(), dqm_interfaces.DQMcommunicator.get_data(), and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.ls().

   def ls_url(self, url):
     url=url.replace(" ","%20")
     url=self.server+url
     #print "listing "+url
     form_folder={}
     raw_folder=None
     try:
       raw_folder=eval(self.get_data(url))
     except:
       print "Retrying.."
       for ntrials in xrange(5):
         try:
           if ntrials!=0:
             sleep(2)
           #raw_folder=loads(self.get_data(url))
           raw_folder=eval(self.get_data(url))
           break
         except:
           print "Could not fetch %s. Retrying" %url
         
     #raw_folder=loads(self.get_data(url))
     for content_dict in raw_folder["contents"]:      
       if "subdir" in content_dict:
         form_folder[content_dict["subdir"]]={"type":'dir'}
       elif "obj" in content_dict:
         properties=content_dict["properties"]
         obj_name=content_dict["obj"]
         obj_type=properties["type"]
         obj_kind=properties["kind"]
         obj_as_string=''
         if "rootobj" in content_dict:
           obj_as_string=content_dict["rootobj"]
         form_folder[obj_name]={'type':obj_type,'obj_as_string':obj_as_string,"kind":obj_kind}
     #for k,v in form_folder.items():
       #print "* %s --> %s" %(k,v["type"])
     
     return form_folder        
   

def dqm_interfaces.DQMcommunicator.open_url	(	self,
		url
	)

Definition at line 74 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.ident, and dqm_interfaces.DQMcommunicator.is_private.

Referenced by dqm_interfaces.DQMcommunicator.get_data().

   def open_url(self,url):
     url=url.replace(' ','%20')
     datareq = Request(url)
     datareq.add_header('User-agent', self.ident)    
     url_obj=0
     if not self.is_private:
       url_obj=self.opener.open(datareq)   
       #url_obj=build_opener(X509CertOpen()).open(datareq) 
     else:
       url_obj=urlopen(datareq)
       
     return url_obj
   