Inheritance diagram for dqm_interfaces.DQMcommunicator:

Public Member Functions
def	__init__ (self, server, is_private=False, ident="DQMToJson/1.0 python/%d.%d.%d"%version_info[:3])

def	cd (self, args)

def	get_common_runs (self, dataset_string1, dataset_string2)

def	get_data (self, full_url)

def	get_dataset_runs (self, dataset_string)

def	get_datasets_list (self, dataset_string="")

def	get_RelVal_CMSSW_versions (self, query)

def	get_root_objects (self, url="")

def	get_root_objects_list (self, url="")

def	get_root_objects_list_recursive (self, url="")

def	get_root_objects_names_list_recursive (self, url="", present_url="")

def	get_root_objects_recursive (self, url="", present_url="")

def	get_runs_list (self, dataset_string)

def	get_samples (self, samples_string="*")

def	ls (self, url='', fetch_root=False)

def	ls_url (self, url)

def	open_url (self, url)

Public Attributes
	DQMpwd

	ident

	is_private

	oldDQMpwd

	opener

	prevDQMpwd

	server

Static Public Attributes
	base_dir

Detailed Description

Communicate with the DQM Document server

Definition at line 53 of file dqm_interfaces.py.

Constructor & Destructor Documentation

def dqm_interfaces.DQMcommunicator.__init__	(	self,
		server,
		is_private = `False`,
		ident = `"DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]`
	)

Definition at line 64 of file dqm_interfaces.py.

                  ident="DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]):
         self.ident = ident
         self.server = server
         self.is_private = is_private
         self.DQMpwd=DQMcommunicator.base_dir
         self.prevDQMpwd=self.DQMpwd
         self.opener=None
         if not self.is_private:
             self.opener=build_opener(X509CertOpen())

Member Function Documentation

def dqm_interfaces.DQMcommunicator.cd	(	self,
		args
	)

Definition at line 157 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.oldDQMpwd, and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DirWalkerFile.walk().

     def cd(self, *args):
         len_args=len(args)
         full_url=""
         if len_args!=1 and len_args!=3:
             raise InvalidNumberOfArguments
         if len_args==3:
             dataset, run, folder = args    
             full_url='%s/data/json/archive/%s/%s/%s' % (self.server, dataset, run, folder)
         if len_args==1:
             folder=args[0]
             if folder==self.DQMpwd:
                 full_url=self.DQMpwd
             elif folder=="..":
                 full_url=self.DQMpwd[:self.DQMpwd.rfind("/")]
             elif folder=="-":
                 full_url=self.oldDQMpwd
             elif folder=="":
                 full_url=DQMcommunicator.base_dir
             else:
                 full_url=self.DQMpwd+"/"+folder
 
         full_url=full_url.replace(' ','%20')
         #print "cd: "+full_url
 
         self.oldDQMpwd=self.DQMpwd
         self.DQMpwd=full_url   
         #print "In %s" %self.DQMpwd
 

def dqm_interfaces.DQMcommunicator.get_common_runs	(	self,
		dataset_string1,
		dataset_string2
	)

Definition at line 240 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_runs_list().

     def get_common_runs(self,dataset_string1,dataset_string2):
         set1=set(self.get_runs_list(dataset_string1))
         set2=set(self.get_runs_list(dataset_string2))
         set1.intersection_update(set2)
         return list (set2)
 

def dqm_interfaces.DQMcommunicator.get_data	(	self,
		full_url
	)

Definition at line 90 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.open_url().

Referenced by dqm_interfaces.DQMcommunicator.get_samples(), and dqm_interfaces.DQMcommunicator.ls_url().

     def get_data(self, full_url):
         #print "getting data from %s" %full_url
         data = self.open_url(full_url).read()
 
         data = sub("-inf", '0', data)
         data = sub("\s+inf", '0', data)
         data = sub("\s+nan", '0', data)
         data = sub('""(CMSSW.*?)""', '"\\1"', data)
 
         return data
 

def dqm_interfaces.DQMcommunicator.get_dataset_runs	(	self,
		dataset_string
	)

Definition at line 232 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_datasets_list(), and dqm_interfaces.DQMcommunicator.get_runs_list().

     def get_dataset_runs(self,dataset_string):
         dataset_runs={}
         for dataset in self.get_datasets_list(dataset_string):
             dataset_runs[dataset]=self.get_runs_list(dataset)
         return dataset_runs
 

def dqm_interfaces.DQMcommunicator.get_datasets_list	(	self,
		dataset_string = `""`
	)

Definition at line 198 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_samples(), and genParticles_cff.map.

Referenced by dqm_interfaces.DQMcommunicator.get_dataset_runs(), and dqm_interfaces.DQMcommunicator.get_RelVal_CMSSW_versions().

     def get_datasets_list(self, dataset_string=""):
         samples_list=self.get_samples(dataset_string)    
         datasets_list=[]
         for sample in samples_list:
             temp_datasets_list =  map(lambda item:item["dataset"] ,sample['items'])
             for temp_dataset in temp_datasets_list:
                 if not temp_dataset in datasets_list:
                     datasets_list.append(temp_dataset)
         return datasets_list
 

def dqm_interfaces.DQMcommunicator.get_RelVal_CMSSW_versions	(	self,
		query
	)

Get the available cmssw versions for the relvals.

Definition at line 210 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_datasets_list(), and list().

     def get_RelVal_CMSSW_versions(self,query):
         """Get the available cmssw versions for the relvals.
         """
         relvals_list=self.get_datasets_list(query)
         # The samples are of the form /RelValTHISISMYFAVOURITECHANNEL/CMSSW_VERSION/GEN-SIM-WHATEVER-RECO
         cmssw_versions_with_duplicates=map (lambda x: x.split("/")[2],relvals_list)
         return list(set(cmssw_versions_with_duplicates))
 

def dqm_interfaces.DQMcommunicator.get_root_objects	(	self,
		url = `""`
	)

Definition at line 262 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, mps_monitormerge.items, ROOTData.literal2root(), ProcLinear.ls, WZInterestingEventSelector::event.ls, ProcMatrix.ls, BeamSpotRcdReader::theBSfromDB.ls, create_public_lumi_plots.LumiDataPoint.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, dqm_interfaces.DQMcommunicator.ls(), and dqm_interfaces.DQMcommunicator.server.

     def get_root_objects(self, url=""):
         if len(url)==0:
             url=self.DQMpwd
         else:
             url=self.server+"/"+url    
         url = url.replace(" ","%20")
         objects={}
         for name,description in self.ls(url,True).items():     
             if "dir" not in description["type"] and "ROOT" in description["kind"]:
                 objects[name]=literal2root(description["obj_as_string"],description["type"])
         return objects
 
      #-------------------------------------------------------------------------------
 

def dqm_interfaces.DQMcommunicator.get_root_objects_list	(	self,
		url = `""`
	)

Definition at line 248 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, mps_monitormerge.items, ROOTData.literal2root(), ProcLinear.ls, WZInterestingEventSelector::event.ls, ProcMatrix.ls, BeamSpotRcdReader::theBSfromDB.ls, create_public_lumi_plots.LumiDataPoint.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

     def get_root_objects_list(self, url=""):
         if len(url)==0:
             url=self.DQMpwd
         else:
             url="/"+url    
         url = url.replace(" ","%20")
         objects=[]
         for name,description in self.ls(url,True).items():     
             if "dir" not in description["type"]  and "ROOT" in description["kind"]:
                 objects.append(literal2root(description["obj_as_string"],description["type"]))
         return objects
 

def dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive	(	self,
		url = `""`
	)

Definition at line 276 of file dqm_interfaces.py.

References TFileDirectory.cd(), TFileService.cd(), DQMStore::IBooker.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), DQMStore::IGetter.cd(), DQMStore.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), mps_monitormerge.items, ROOTData.literal2root(), ProcLinear.ls, WZInterestingEventSelector::event.ls, ProcMatrix.ls, BeamSpotRcdReader::theBSfromDB.ls, create_public_lumi_plots.LumiDataPoint.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive().

     def get_root_objects_list_recursive(self, url=""):
         null_url = (len(url)==0)    
         if len(url)==0:
             url=self.DQMpwd
         else:
             url="/"+url    
         url = url.replace(" ","%20")      
         if not null_url: 
             self.cd(url)
         objects=[]
         for name,description in self.ls("",True).items():     
             if "dir" in description["type"]:
                 objects+=self.get_root_objects_list_recursive(name)
                 self.cd("..")
             elif  "ROOT" in description["kind"]:
                 objects.append(literal2root(description["obj_as_string"],description["type"]))
         if not null_url: 
             self.cd("..")
         return objects
 
      #-------------------------------------------------------------------------------
 

def dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive	(	self,
		url = `""`,
		present_url = `""`
	)

Definition at line 298 of file dqm_interfaces.py.

References TFileDirectory.cd(), TFileService.cd(), DQMStore::IBooker.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), DQMStore::IGetter.cd(), DQMStore.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), mps_monitormerge.items, ProcLinear.ls, WZInterestingEventSelector::event.ls, ProcMatrix.ls, BeamSpotRcdReader::theBSfromDB.ls, create_public_lumi_plots.LumiDataPoint.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive().

     def get_root_objects_names_list_recursive(self, url="",present_url=""):
         null_url = (len(url)==0)
         if (not null_url):
             if len(present_url)==0:
                 present_url=url
             else:
                 present_url+="_%s"%url
         if len(url)==0:
             url=self.DQMpwd
         else:
             url="/"+url    
         url = url.replace(" ","%20")
         if not null_url:
             self.cd(url)
         objects_names=[]
         for name,description in self.ls("",False).items():     
             if "dir" in description["type"]:        
                 objects_names+=self.get_root_objects_names_list_recursive(name,present_url)
                 self.cd("..")
             elif  "ROOT" in description["kind"]:
                 objects_names.append("%s_%s"%(present_url,name))
         if not null_url: 
             self.cd("..")
         return objects_names
 
      #-------------------------------------------------------------------------------
 

def dqm_interfaces.DQMcommunicator.get_root_objects_recursive	(	self,
		url = `""`,
		present_url = `""`
	)

Definition at line 325 of file dqm_interfaces.py.

References TFileDirectory.cd(), TFileService.cd(), DQMStore::IBooker.cd(), Folder.cd(), dqm_interfaces.DQMcommunicator.cd(), DQMStore::IGetter.cd(), DQMStore.cd(), python.rootplot.utilities.RootFile.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), mps_monitormerge.items, ROOTData.literal2root(), ProcLinear.ls, WZInterestingEventSelector::event.ls, ProcMatrix.ls, BeamSpotRcdReader::theBSfromDB.ls, create_public_lumi_plots.LumiDataPoint.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_recursive().

     def get_root_objects_recursive(self, url="",present_url=""):
         null_url = (len(url)==0)
         if (not null_url):
             if len(present_url)==0:
                 present_url=url
             else:
                 present_url+="_%s"%url
         if len(url)==0:
             url=self.DQMpwd
         else:
             url="/"+url    
         url = url.replace(" ","%20")
         #if not null_url:
         self.cd(url)
         objects={}
         for name,description in self.ls("",True).items():     
             if "dir" in description["type"]:
                 objects.update(self.get_root_objects_recursive(name,present_url))
                 self.cd("..")
             elif  "ROOT" in description["kind"]:
                 objects["%s_%s"%(present_url,name)]=literal2root(description["obj_as_string"],description["type"])
         #if not null_url:
         self.cd("..")
         return objects
 
 #-------------------------------------------------------------------------------
 

def dqm_interfaces.DQMcommunicator.get_runs_list	(	self,
		dataset_string
	)

Definition at line 220 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_samples(), and genParticles_cff.map.

Referenced by dqm_interfaces.DQMcommunicator.get_common_runs(), and dqm_interfaces.DQMcommunicator.get_dataset_runs().

     def get_runs_list(self, dataset_string):
         slash="/"
         while(dataset_string.endswith(slash) or dataset_string.beginswith(slash)):
             dataset_string=dataset_string.strip("/")
         samples_list=self.get_samples(dataset_string)
         runlist=[]
         # Get all the runs in all the items which are in every sample
         map( lambda sample: map (lambda item: runlist.append(item['run']), sample['items']), samples_list)
         return runlist
 

def dqm_interfaces.DQMcommunicator.get_samples	(	self,
		samples_string = `"*"`
	)

A sample contains, among the other things, a data type, a dataset name 
and a run.

Definition at line 187 of file dqm_interfaces.py.

References data_sources.data_source.get_data(), dqm_interfaces.DQMcommunicator.get_data(), and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.get_datasets_list(), and dqm_interfaces.DQMcommunicator.get_runs_list().

     def get_samples(self, samples_string="*"):
         """
         A sample contains, among the other things, a data type, a dataset name 
         and a run.
         """
         full_url='%s/data/json/samples?match=%s' % (self.server, samples_string)
         samples_dict=eval(self.get_data(full_url))
         return samples_dict["samples"]
 

def dqm_interfaces.DQMcommunicator.ls	(	self,
		url = `''`,
		fetch_root = `False`
	)

Definition at line 143 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, join(), and dqm_interfaces.DQMcommunicator.ls_url().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects(), dqm_interfaces.DQMcommunicator.get_root_objects_list(), dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DirWalkerFile.walk().

     def ls(self, url='', fetch_root=False):
         if len(url)==0:
             url=join(self.DQMpwd,url)
 
         form_folder={}   
 
         if fetch_root:
             url='%s?rootcontent=1'%url
         form_folder=self.ls_url(url)
 
         return form_folder
 

def dqm_interfaces.DQMcommunicator.ls_url	(	self,
		url
	)

Definition at line 103 of file dqm_interfaces.py.

References data_sources.data_source.get_data(), dqm_interfaces.DQMcommunicator.get_data(), edm.print(), and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.ls().

     def ls_url(self, url):
         url=url.replace(" ","%20")
         url=self.server+url
         #print "listing "+url
         form_folder={}
         raw_folder=None
         try:
             raw_folder=eval(self.get_data(url))
         except:
             print("Retrying..")
             for ntrials in xrange(5):
                 try:
                     if ntrials!=0:
                         sleep(2)
                     #raw_folder=loads(self.get_data(url))
                     raw_folder=eval(self.get_data(url))
                     break
                 except:
                     print("Could not fetch %s. Retrying" %url)
 
         #raw_folder=loads(self.get_data(url))
         for content_dict in raw_folder["contents"]:      
             if "subdir" in content_dict:
                 form_folder[content_dict["subdir"]]={"type":'dir'}
             elif "obj" in content_dict:
                 properties=content_dict["properties"]
                 obj_name=content_dict["obj"]
                 obj_type=properties["type"]
                 obj_kind=properties["kind"]
                 obj_as_string=''
                 if "rootobj" in content_dict:
                     obj_as_string=content_dict["rootobj"]
                 form_folder[obj_name]={'type':obj_type,'obj_as_string':obj_as_string,"kind":obj_kind}
         #for k,v in form_folder.items():
             #print "* %s --> %s" %(k,v["type"])
 
         return form_folder        
 

def dqm_interfaces.DQMcommunicator.open_url	(	self,
		url
	)

Definition at line 75 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.ident, and dqm_interfaces.DQMcommunicator.is_private.

Referenced by dqm_interfaces.DQMcommunicator.get_data().

     def open_url(self,url):
         url=url.replace(' ','%20')
         datareq = Request(url)
         datareq.add_header('User-agent', self.ident)    
         url_obj=0
         if not self.is_private:
             url_obj=self.opener.open(datareq)   
             #url_obj=build_opener(X509CertOpen()).open(datareq) 
         else:
             url_obj=urlopen(datareq)
 
         return url_obj
 