Public Member Functions
def	__init__
def	cd
def	get_common_runs
def	get_data
def	get_dataset_runs
def	get_datasets_list
def	get_RelVal_CMSSW_versions
def	get_root_objects
def	get_root_objects_list
def	get_root_objects_list_recursive
def	get_root_objects_names_list_recursive
def	get_root_objects_recursive
def	get_runs_list
def	get_samples
def	ls
def	ls_url
def	open_url
Public Attributes
	DQMpwd
	ident
	is_private
	oldDQMpwd
	opener
	prevDQMpwd
	server
Static Public Attributes
string	base_dir = '/data/json/archive/'

Detailed Description

Communicate with the DQM Document server

Definition at line 55 of file dqm_interfaces.py.

Constructor & Destructor Documentation

def dqm_interfaces::DQMcommunicator::__init__	(	self,
		server,
		is_private = `False`,
		ident = `"DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]`
	)

Definition at line 63 of file dqm_interfaces.py.

00066                                                                     :3]):
00067     self.ident = ident
00068     self.server = server
00069     self.is_private = is_private
00070     self.DQMpwd=DQMcommunicator.base_dir
00071     self.prevDQMpwd=self.DQMpwd
00072     self.opener=None
00073     if not self.is_private:
00074       self.opener=build_opener(X509CertOpen())

Member Function Documentation

def dqm_interfaces::DQMcommunicator::cd	(	self,
		args
	)

Definition at line 159 of file dqm_interfaces.py.

00160                      :
00161     len_args=len(args)
00162     full_url=""
00163     if len_args!=1 and len_args!=3:
00164       raise(InvalidNumberOfArguments("3 or 1 args expected!"))
00165     if len_args==3:
00166       dataset, run, folder = args    
00167       full_url='%s/data/json/archive/%s/%s/%s' % (self.server, dataset, run, folder)
00168     if len_args==1:
00169       folder=args[0]
00170       if folder==self.DQMpwd:
00171         full_url=self.DQMpwd
00172       elif folder=="..":
00173         full_url=self.DQMpwd[:self.DQMpwd.rfind("/")]
00174       elif folder=="-":
00175         full_url=self.oldDQMpwd
00176       elif folder=="":
00177         full_url=DQMcommunicator.base_dir
00178       else:
00179         full_url=self.DQMpwd+"/"+folder
00180         
00181     full_url=full_url.replace(' ','%20')
00182     #print "cd: "+full_url
00183     
00184     self.oldDQMpwd=self.DQMpwd
00185     self.DQMpwd=full_url   
00186     #print "In %s" %self.DQMpwd

def dqm_interfaces::DQMcommunicator::get_common_runs	(	self,
		dataset_string1,
		dataset_string2
	)

Definition at line 242 of file dqm_interfaces.py.

00243                                                            :
00244     set1=set(self.get_runs_list(dataset_string1))
00245     set2=set(self.get_runs_list(dataset_string2))
00246     set1.intersection_update(set2)
00247     return list (set2)

def dqm_interfaces::DQMcommunicator::get_data	(	self,
		full_url
	)

Definition at line 92 of file dqm_interfaces.py.

00093                               :
00094     #print "getting data from %s" %full_url
00095     data = self.open_url(full_url).read()
00096     
00097     data = sub("-inf", '0', data)
00098     data = sub("\s+inf", '0', data)
00099     data = sub("\s+nan", '0', data)
00100     data = sub('""(CMSSW.*?)""', '"\\1"', data)
00101    
00102     return data

def dqm_interfaces::DQMcommunicator::get_dataset_runs	(	self,
		dataset_string
	)

Definition at line 234 of file dqm_interfaces.py.

00235                                            :
00236     dataset_runs={}
00237     for dataset in self.get_datasets_list(dataset_string):
00238       dataset_runs[dataset]=self.get_runs_list(dataset)
00239     return dataset_runs

def dqm_interfaces::DQMcommunicator::get_datasets_list	(	self,
		dataset_string = `""`
	)

Definition at line 200 of file dqm_interfaces.py.

00201                                                 :
00202     samples_list=self.get_samples(dataset_string)    
00203     datasets_list=[]
00204     for sample in samples_list:
00205       temp_datasets_list =  map(lambda item:item["dataset"] ,sample['items'])
00206       for temp_dataset in temp_datasets_list:
00207         if not temp_dataset in datasets_list:
00208           datasets_list.append(temp_dataset)
00209     return datasets_list

def dqm_interfaces::DQMcommunicator::get_RelVal_CMSSW_versions	(	self,
		query
	)

Get the available cmssw versions for the relvals.

Definition at line 212 of file dqm_interfaces.py.

00213                                            :
00214     """Get the available cmssw versions for the relvals.
00215     """
00216     relvals_list=self.get_datasets_list(query)
00217     # The samples are of the form /RelValTHISISMYFAVOURITECHANNEL/CMSSW_VERSION/GEN-SIM-WHATEVER-RECO
00218     cmssw_versions_with_duplicates=map (lambda x: x.split("/")[2],relvals_list)
00219     return list(set(cmssw_versions_with_duplicates))

def dqm_interfaces::DQMcommunicator::get_root_objects	(	self,
		url = `""`
	)

Definition at line 264 of file dqm_interfaces.py.

00265                                     :
00266     if len(url)==0:
00267       url=self.DQMpwd
00268     else:
00269       url=self.server+"/"+url    
00270     url = url.replace(" ","%20")
00271     objects={}
00272     for name,description in self.ls(url,True).items():     
00273       if "dir" not in description["type"] and "ROOT" in description["kind"]:
00274         objects[name]=literal2root(description["obj_as_string"],description["type"])
00275     return objects

def dqm_interfaces::DQMcommunicator::get_root_objects_list	(	self,
		url = `""`
	)

Definition at line 250 of file dqm_interfaces.py.

00251                                          :
00252     if len(url)==0:
00253       url=self.DQMpwd
00254     else:
00255       url="/"+url    
00256     url = url.replace(" ","%20")
00257     objects=[]
00258     for name,description in self.ls(url,True).items():     
00259       if "dir" not in description["type"]  and "ROOT" in description["kind"]:
00260         objects.append(literal2root(description["obj_as_string"],description["type"]))
00261     return objects

def dqm_interfaces::DQMcommunicator::get_root_objects_list_recursive	(	self,
		url = `""`
	)

Definition at line 278 of file dqm_interfaces.py.

00279                                                    :
00280     null_url = (len(url)==0)    
00281     if len(url)==0:
00282       url=self.DQMpwd
00283     else:
00284       url="/"+url    
00285     url = url.replace(" ","%20")      
00286     if not null_url: 
00287       self.cd(url)
00288     objects=[]
00289     for name,description in self.ls("",True).items():     
00290       if "dir" in description["type"]:
00291         objects+=self.get_root_objects_list_recursive(name)
00292         self.cd("..")
00293       elif  "ROOT" in description["kind"]:
00294         objects.append(literal2root(description["obj_as_string"],description["type"]))
00295     if not null_url: 
00296       self.cd("..")
00297     return objects

def dqm_interfaces::DQMcommunicator::get_root_objects_names_list_recursive	(	self,
		url = `""`,
		present_url = `""`
	)

Definition at line 300 of file dqm_interfaces.py.

00301                                                                         :
00302     null_url = (len(url)==0)
00303     if (not null_url):
00304       if len(present_url)==0:
00305         present_url=url
00306       else:
00307         present_url+="_%s"%url
00308     if len(url)==0:
00309       url=self.DQMpwd
00310     else:
00311       url="/"+url    
00312     url = url.replace(" ","%20")
00313     if not null_url:
00314       self.cd(url)
00315     objects_names=[]
00316     for name,description in self.ls("",False).items():     
00317       if "dir" in description["type"]:        
00318         objects_names+=self.get_root_objects_names_list_recursive(name,present_url)
00319         self.cd("..")
00320       elif  "ROOT" in description["kind"]:
00321         objects_names.append("%s_%s"%(present_url,name))
00322     if not null_url: 
00323       self.cd("..")
00324     return objects_names

def dqm_interfaces::DQMcommunicator::get_root_objects_recursive	(	self,
		url = `""`,
		present_url = `""`
	)

Definition at line 327 of file dqm_interfaces.py.

00328                                                              :
00329     null_url = (len(url)==0)
00330     if (not null_url):
00331       if len(present_url)==0:
00332         present_url=url
00333       else:
00334         present_url+="_%s"%url
00335     if len(url)==0:
00336       url=self.DQMpwd
00337     else:
00338       url="/"+url    
00339     url = url.replace(" ","%20")
00340     #if not null_url:
00341     self.cd(url)
00342     objects={}
00343     for name,description in self.ls("",True).items():     
00344       if "dir" in description["type"]:
00345         objects.update(self.get_root_objects_recursive(name,present_url))
00346         self.cd("..")
00347       elif  "ROOT" in description["kind"]:
00348         objects["%s_%s"%(present_url,name)]=literal2root(description["obj_as_string"],description["type"])
00349     #if not null_url:
00350     self.cd("..")
00351     return objects
00352 
00353 #-------------------------------------------------------------------------------

def dqm_interfaces::DQMcommunicator::get_runs_list	(	self,
		dataset_string
	)

Definition at line 222 of file dqm_interfaces.py.

00223                                          :
00224     slash="/"
00225     while(dataset_string.endswith(slash) or dataset_string.beginswith(slash)):
00226       dataset_string=dataset_string.strip("/")
00227     samples_list=self.get_samples(dataset_string)
00228     runlist=[]
00229     # Get all the runs in all the items which are in every sample
00230     map( lambda sample: map (lambda item: runlist.append(item['run']), sample['items']), samples_list)
00231     return runlist

def dqm_interfaces::DQMcommunicator::get_samples	(	self,
		samples_string = `"*"`
	)

A sample contains, among the other things, a data type, a dataset name 
and a run.

Definition at line 189 of file dqm_interfaces.py.

00190                                            :
00191     """
00192     A sample contains, among the other things, a data type, a dataset name 
00193     and a run.
00194     """
00195     full_url='%s/data/json/samples?match=%s' % (self.server, samples_string)
00196     samples_dict=eval(self.get_data(full_url))
00197     return samples_dict["samples"]

def dqm_interfaces::DQMcommunicator::ls	(	self,
		url = `''`,
		fetch_root = `False`
	)

Definition at line 145 of file dqm_interfaces.py.

00146                                         :
00147     if len(url)==0:
00148       url=join(self.DQMpwd,url)
00149     
00150     form_folder={}   
00151     
00152     if fetch_root:
00153       url='%s?rootcontent=1'%url
00154     form_folder=self.ls_url(url)
00155 
00156     return form_folder

def dqm_interfaces::DQMcommunicator::ls_url	(	self,
		url
	)

Definition at line 105 of file dqm_interfaces.py.

00106                        :
00107     url=url.replace(" ","%20")
00108     url=self.server+url
00109     #print "listing "+url
00110     form_folder={}
00111     raw_folder=None
00112     try:
00113       raw_folder=eval(self.get_data(url))
00114     except:
00115       print "Retrying.."
00116       for ntrials in xrange(5):
00117         try:
00118           if ntrials!=0:
00119             sleep(2)
00120           #raw_folder=loads(self.get_data(url))
00121           raw_folder=eval(self.get_data(url))
00122           break
00123         except:
00124           print "Could not fetch %s. Retrying" %url
00125         
00126     #raw_folder=loads(self.get_data(url))
00127     for content_dict in raw_folder["contents"]:      
00128       if content_dict.has_key("subdir"):
00129         form_folder[content_dict["subdir"]]={"type":'dir'}
00130       elif content_dict.has_key("obj"):
00131         properties=content_dict["properties"]
00132         obj_name=content_dict["obj"]
00133         obj_type=properties["type"]
00134         obj_kind=properties["kind"]
00135         obj_as_string=''
00136         if content_dict.has_key("rootobj"):
00137           obj_as_string=content_dict["rootobj"]
00138         form_folder[obj_name]={'type':obj_type,'obj_as_string':obj_as_string,"kind":obj_kind}
00139     #for k,v in form_folder.items():
00140       #print "* %s --> %s" %(k,v["type"])
00141     
00142     return form_folder

def dqm_interfaces::DQMcommunicator::open_url	(	self,
		url
	)

Definition at line 77 of file dqm_interfaces.py.

00078                         :
00079     url=url.replace(' ','%20')
00080     datareq = Request(url)
00081     datareq.add_header('User-agent', self.ident)    
00082     url_obj=0
00083     if not self.is_private:
00084       url_obj=self.opener.open(datareq)   
00085       #url_obj=build_opener(X509CertOpen()).open(datareq) 
00086     else:
00087       url_obj=urlopen(datareq)
00088       
00089     return url_obj