dqm_interfaces.DQMcommunicator Class Reference

Inheritance diagram for dqm_interfaces.DQMcommunicator:

Public Member Functions
def	__init__ (self, server, is_private=False, ident="DQMToJson/1.0 python/%d.%d.%d" % version_info[:3])
def	cd (self, args)
def	get_common_runs (self, dataset_string1, dataset_string2)
def	get_data (self, full_url)
def	get_dataset_runs (self, dataset_string)
def	get_datasets_list (self, dataset_string="")
def	get_RelVal_CMSSW_versions (self, query)
def	get_root_objects (self, url="")
def	get_root_objects_list (self, url="")
def	get_root_objects_list_recursive (self, url="")
def	get_root_objects_names_list_recursive (self, url="", present_url="")
def	get_root_objects_recursive (self, url="", present_url="")
def	get_runs_list (self, dataset_string)
def	get_samples (self, samples_string="*")
def	ls (self, url='', fetch_root=False)
def	ls_url (self, url)
def	open_url (self, url)

Public Attributes
	DQMpwd
	ident
	is_private
	oldDQMpwd
	opener
	prevDQMpwd
	server

Static Public Attributes
	base_dir

Detailed Description

Communicate with the DQM Document server

Definition at line 57 of file dqm_interfaces.py.

Constructor & Destructor Documentation

__init__()

def dqm_interfaces.DQMcommunicator.__init__	(		self,
			server,
			is_private = False,
			ident = "DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]
	)

Definition at line 68 of file dqm_interfaces.py.

                 ident="DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]):
        self.ident = ident
        self.server = server
        self.is_private = is_private
        self.DQMpwd=DQMcommunicator.base_dir
        self.prevDQMpwd=self.DQMpwd
        self.opener=None
        if not self.is_private:
            self.opener=build_opener(X509CertOpen())

Member Function Documentation

cd()

def dqm_interfaces.DQMcommunicator.cd	(		self,
			args
	)

Definition at line 161 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.oldDQMpwd, and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DirWalkerFile.walk().

    def cd(self, *args):
        len_args=len(args)
        full_url=""
        if len_args!=1 and len_args!=3:
            raise InvalidNumberOfArguments
        if len_args==3:
            dataset, run, folder = args    
            full_url='%s/data/json/archive/%s/%s/%s' % (self.server, dataset, run, folder)
        if len_args==1:
            folder=args[0]
            if folder==self.DQMpwd:
                full_url=self.DQMpwd
            elif folder=="..":
                full_url=self.DQMpwd[:self.DQMpwd.rfind("/")]
            elif folder=="-":
                full_url=self.oldDQMpwd
            elif folder=="":
                full_url=DQMcommunicator.base_dir
            else:
                full_url=self.DQMpwd+"/"+folder

        full_url=full_url.replace(' ','%20')
        #print "cd: "+full_url

        self.oldDQMpwd=self.DQMpwd
        self.DQMpwd=full_url   
        #print "In %s" %self.DQMpwd

get_common_runs()

def dqm_interfaces.DQMcommunicator.get_common_runs	(		self,
			dataset_string1,
			dataset_string2
	)

Definition at line 244 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_runs_list().

    def get_common_runs(self,dataset_string1,dataset_string2):
        set1=set(self.get_runs_list(dataset_string1))
        set2=set(self.get_runs_list(dataset_string2))
        set1.intersection_update(set2)
        return list (set2)

get_data()

def dqm_interfaces.DQMcommunicator.get_data	(		self,
			full_url
	)

Definition at line 94 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.open_url(), and fileinputsource_cfi.read.

Referenced by dqm_interfaces.DQMcommunicator.get_samples(), and dqm_interfaces.DQMcommunicator.ls_url().

    def get_data(self, full_url):
        #print "getting data from %s" %full_url
        data = self.open_url(full_url).read()

        data = sub("-inf", '0', data)
        data = sub("\s+inf", '0', data)
        data = sub("\s+nan", '0', data)
        data = sub('""(CMSSW.*?)""', '"\\1"', data)

        return data

get_dataset_runs()

def dqm_interfaces.DQMcommunicator.get_dataset_runs	(		self,
			dataset_string
	)

Definition at line 236 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_datasets_list(), and dqm_interfaces.DQMcommunicator.get_runs_list().

    def get_dataset_runs(self,dataset_string):
        dataset_runs={}
        for dataset in self.get_datasets_list(dataset_string):
            dataset_runs[dataset]=self.get_runs_list(dataset)
        return dataset_runs

get_datasets_list()

def dqm_interfaces.DQMcommunicator.get_datasets_list	(		self,
			dataset_string = ""
	)

Definition at line 202 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_samples(), and genParticles_cff.map.

Referenced by dqm_interfaces.DQMcommunicator.get_dataset_runs(), and dqm_interfaces.DQMcommunicator.get_RelVal_CMSSW_versions().

    def get_datasets_list(self, dataset_string=""):
        samples_list=self.get_samples(dataset_string)    
        datasets_list=[]
        for sample in samples_list:
            temp_datasets_list =  map(lambda item:item["dataset"] ,sample['items'])
            for temp_dataset in temp_datasets_list:
                if not temp_dataset in datasets_list:
                    datasets_list.append(temp_dataset)
        return datasets_list

get_RelVal_CMSSW_versions()

def dqm_interfaces.DQMcommunicator.get_RelVal_CMSSW_versions	(		self,
			query
	)

Get the available cmssw versions for the relvals.

Definition at line 214 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_datasets_list().

    def get_RelVal_CMSSW_versions(self,query):
        """Get the available cmssw versions for the relvals.
        """
        relvals_list=self.get_datasets_list(query)
        # The samples are of the form /RelValTHISISMYFAVOURITECHANNEL/CMSSW_VERSION/GEN-SIM-WHATEVER-RECO
        cmssw_versions_with_duplicates=map (lambda x: x.split("/")[2],relvals_list)
        return list(set(cmssw_versions_with_duplicates))

get_root_objects()

def dqm_interfaces.DQMcommunicator.get_root_objects	(		self,
			url = ""
	)

Definition at line 266 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, mps_monitormerge.items, ROOTData.literal2root(), WZInterestingEventSelector::event.ls, BeamProfile2DBReader::TheBSfromDB.ls, BeamSpotRcdReader::theBSfromDB.ls, BeamSpotOnlineRecordsReader::theBSOfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, dqm_interfaces.DQMcommunicator.ls(), and dqm_interfaces.DQMcommunicator.server.

    def get_root_objects(self, url=""):
        if len(url)==0:
            url=self.DQMpwd
        else:
            url=self.server+"/"+url    
        url = url.replace(" ","%20")
        objects={}
        for name,description in self.ls(url,True).items():     
            if "dir" not in description["type"] and "ROOT" in description["kind"]:
                objects[name]=literal2root(description["obj_as_string"],description["type"])
        return objects

     #-------------------------------------------------------------------------------

get_root_objects_list()

def dqm_interfaces.DQMcommunicator.get_root_objects_list	(		self,
			url = ""
	)

Definition at line 252 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, mps_monitormerge.items, ROOTData.literal2root(), WZInterestingEventSelector::event.ls, BeamProfile2DBReader::TheBSfromDB.ls, BeamSpotRcdReader::theBSfromDB.ls, BeamSpotOnlineRecordsReader::theBSOfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

    def get_root_objects_list(self, url=""):
        if len(url)==0:
            url=self.DQMpwd
        else:
            url="/"+url    
        url = url.replace(" ","%20")
        objects=[]
        for name,description in self.ls(url,True).items():     
            if "dir" not in description["type"]  and "ROOT" in description["kind"]:
                objects.append(literal2root(description["obj_as_string"],description["type"]))
        return objects

get_root_objects_list_recursive()

def dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive	(		self,
			url = ""
	)

Definition at line 280 of file dqm_interfaces.py.

References dqm::implementation::NavigatorBase.cd(), TFileDirectory.cd(), TFileService.cd(), dqm_interfaces.DQMcommunicator.cd(), Folder.cd(), mkfit::ConfigJsonPatcher.cd(), python.rootplot.utilities.RootFile.cd(), dqm::implementation::DQMStore.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), mps_monitormerge.items, ROOTData.literal2root(), WZInterestingEventSelector::event.ls, BeamProfile2DBReader::TheBSfromDB.ls, BeamSpotRcdReader::theBSfromDB.ls, BeamSpotOnlineRecordsReader::theBSOfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive().

    def get_root_objects_list_recursive(self, url=""):
        null_url = (len(url)==0)    
        if len(url)==0:
            url=self.DQMpwd
        else:
            url="/"+url    
        url = url.replace(" ","%20")      
        if not null_url: 
            self.cd(url)
        objects=[]
        for name,description in self.ls("",True).items():     
            if "dir" in description["type"]:
                objects+=self.get_root_objects_list_recursive(name)
                self.cd("..")
            elif  "ROOT" in description["kind"]:
                objects.append(literal2root(description["obj_as_string"],description["type"]))
        if not null_url: 
            self.cd("..")
        return objects

     #-------------------------------------------------------------------------------

get_root_objects_names_list_recursive()

def dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive	(		self,
			url = "",
			present_url = ""
	)

Definition at line 302 of file dqm_interfaces.py.

References dqm::implementation::NavigatorBase.cd(), TFileDirectory.cd(), TFileService.cd(), dqm_interfaces.DQMcommunicator.cd(), Folder.cd(), mkfit::ConfigJsonPatcher.cd(), python.rootplot.utilities.RootFile.cd(), dqm::implementation::DQMStore.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), mps_monitormerge.items, WZInterestingEventSelector::event.ls, BeamProfile2DBReader::TheBSfromDB.ls, BeamSpotRcdReader::theBSfromDB.ls, BeamSpotOnlineRecordsReader::theBSOfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive().

    def get_root_objects_names_list_recursive(self, url="",present_url=""):
        null_url = (len(url)==0)
        if (not null_url):
            if len(present_url)==0:
                present_url=url
            else:
                present_url+="_%s"%url
        if len(url)==0:
            url=self.DQMpwd
        else:
            url="/"+url    
        url = url.replace(" ","%20")
        if not null_url:
            self.cd(url)
        objects_names=[]
        for name,description in self.ls("",False).items():     
            if "dir" in description["type"]:        
                objects_names+=self.get_root_objects_names_list_recursive(name,present_url)
                self.cd("..")
            elif  "ROOT" in description["kind"]:
                objects_names.append("%s_%s"%(present_url,name))
        if not null_url: 
            self.cd("..")
        return objects_names

     #-------------------------------------------------------------------------------

get_root_objects_recursive()

def dqm_interfaces.DQMcommunicator.get_root_objects_recursive	(		self,
			url = "",
			present_url = ""
	)

Definition at line 329 of file dqm_interfaces.py.

References dqm::implementation::NavigatorBase.cd(), TFileDirectory.cd(), TFileService.cd(), dqm_interfaces.DQMcommunicator.cd(), Folder.cd(), mkfit::ConfigJsonPatcher.cd(), python.rootplot.utilities.RootFile.cd(), dqm::implementation::DQMStore.cd(), dqm_interfaces.DQMcommunicator.DQMpwd, dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), mps_monitormerge.items, ROOTData.literal2root(), WZInterestingEventSelector::event.ls, BeamProfile2DBReader::TheBSfromDB.ls, BeamSpotRcdReader::theBSfromDB.ls, BeamSpotOnlineRecordsReader::theBSOfromDB.ls, MatrixUtil.InputInfo.ls, EcalDeadCellDeltaRFilter.ls, and dqm_interfaces.DQMcommunicator.ls().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects_recursive().

    def get_root_objects_recursive(self, url="",present_url=""):
        null_url = (len(url)==0)
        if (not null_url):
            if len(present_url)==0:
                present_url=url
            else:
                present_url+="_%s"%url
        if len(url)==0:
            url=self.DQMpwd
        else:
            url="/"+url    
        url = url.replace(" ","%20")
        #if not null_url:
        self.cd(url)
        objects={}
        for name,description in self.ls("",True).items():     
            if "dir" in description["type"]:
                objects.update(self.get_root_objects_recursive(name,present_url))
                self.cd("..")
            elif  "ROOT" in description["kind"]:
                objects["%s_%s"%(present_url,name)]=literal2root(description["obj_as_string"],description["type"])
        #if not null_url:
        self.cd("..")
        return objects

#-------------------------------------------------------------------------------

get_runs_list()

def dqm_interfaces.DQMcommunicator.get_runs_list	(		self,
			dataset_string
	)

Definition at line 224 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.get_samples(), genParticles_cff.map, and gpuVertexFinder.while().

Referenced by dqm_interfaces.DQMcommunicator.get_common_runs(), and dqm_interfaces.DQMcommunicator.get_dataset_runs().

    def get_runs_list(self, dataset_string):
        slash="/"
        while(dataset_string.endswith(slash) or dataset_string.beginswith(slash)):
            dataset_string=dataset_string.strip("/")
        samples_list=self.get_samples(dataset_string)
        runlist=[]
        # Get all the runs in all the items which are in every sample
        map( lambda sample: map (lambda item: runlist.append(item['run']), sample['items']), samples_list)
        return runlist

get_samples()

def dqm_interfaces.DQMcommunicator.get_samples	(		self,
			samples_string = "*"
	)

A sample contains, among the other things, a data type, a dataset name 
and a run.

Definition at line 191 of file dqm_interfaces.py.

References data_sources.data_source.get_data(), dqm_interfaces.DQMcommunicator.get_data(), and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.get_datasets_list(), and dqm_interfaces.DQMcommunicator.get_runs_list().

    def get_samples(self, samples_string="*"):
        """
        A sample contains, among the other things, a data type, a dataset name 
        and a run.
        """
        full_url='%s/data/json/samples?match=%s' % (self.server, samples_string)
        samples_dict=eval(self.get_data(full_url))
        return samples_dict["samples"]

ls()

def dqm_interfaces.DQMcommunicator.ls	(		self,
			url = '',
			fetch_root = False
	)

Definition at line 147 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.DQMpwd, join(), and dqm_interfaces.DQMcommunicator.ls_url().

Referenced by dqm_interfaces.DQMcommunicator.get_root_objects(), dqm_interfaces.DQMcommunicator.get_root_objects_list(), dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DirWalkerFile.walk().

    def ls(self, url='', fetch_root=False):
        if len(url)==0:
            url=join(self.DQMpwd,url)

        form_folder={}   

        if fetch_root:
            url='%s?rootcontent=1'%url
        form_folder=self.ls_url(url)

        return form_folder

ls_url()

def dqm_interfaces.DQMcommunicator.ls_url	(		self,
			url
	)

Definition at line 107 of file dqm_interfaces.py.

References data_sources.data_source.get_data(), dqm_interfaces.DQMcommunicator.get_data(), print(), FastTimerService_cff.range, and dqm_interfaces.DQMcommunicator.server.

Referenced by dqm_interfaces.DQMcommunicator.ls().

    def ls_url(self, url):
        url=url.replace(" ","%20")
        url=self.server+url
        #print "listing "+url
        form_folder={}
        raw_folder=None
        try:
            raw_folder=eval(self.get_data(url))
        except:
            print("Retrying..")
            for ntrials in range(5):
                try:
                    if ntrials!=0:
                        sleep(2)
                    #raw_folder=loads(self.get_data(url))
                    raw_folder=eval(self.get_data(url))
                    break
                except:
                    print("Could not fetch %s. Retrying" %url)

        #raw_folder=loads(self.get_data(url))
        for content_dict in raw_folder["contents"]:      
            if "subdir" in content_dict:
                form_folder[content_dict["subdir"]]={"type":'dir'}
            elif "obj" in content_dict:
                properties=content_dict["properties"]
                obj_name=content_dict["obj"]
                obj_type=properties["type"]
                obj_kind=properties["kind"]
                obj_as_string=''
                if "rootobj" in content_dict:
                    obj_as_string=content_dict["rootobj"]
                form_folder[obj_name]={'type':obj_type,'obj_as_string':obj_as_string,"kind":obj_kind}
        #for k,v in form_folder.items():
            #print "* %s --> %s" %(k,v["type"])

        return form_folder        

open_url()

def dqm_interfaces.DQMcommunicator.open_url	(		self,
			url
	)

Definition at line 79 of file dqm_interfaces.py.

References dqm_interfaces.DQMcommunicator.ident, dqm_interfaces.DQMcommunicator.is_private, and dqm_interfaces.DQMcommunicator.opener.

Referenced by dqm_interfaces.DQMcommunicator.get_data().

    def open_url(self,url):
        url=url.replace(' ','%20')
        datareq = Request(url)
        datareq.add_header('User-agent', self.ident)    
        url_obj=0
        if not self.is_private:
            url_obj=self.opener.open(datareq)   
            #url_obj=build_opener(X509CertOpen()).open(datareq) 
        else:
            url_obj=urlopen(datareq)

        return url_obj

Member Data Documentation

base_dir

dqm_interfaces.DQMcommunicator.base_dir

static

Definition at line 63 of file dqm_interfaces.py.

DQMpwd

dqm_interfaces.DQMcommunicator.DQMpwd

Definition at line 72 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.cd(), dqm_interfaces.DQMcommunicator.get_root_objects(), dqm_interfaces.DQMcommunicator.get_root_objects_list(), dqm_interfaces.DQMcommunicator.get_root_objects_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_names_list_recursive(), dqm_interfaces.DQMcommunicator.get_root_objects_recursive(), and dqm_interfaces.DQMcommunicator.ls().

ident

dqm_interfaces.DQMcommunicator.ident

Definition at line 69 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.open_url().

is_private

dqm_interfaces.DQMcommunicator.is_private

Definition at line 71 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.open_url().

oldDQMpwd

dqm_interfaces.DQMcommunicator.oldDQMpwd

Definition at line 185 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.cd().

opener

dqm_interfaces.DQMcommunicator.opener

Definition at line 74 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.open_url().

prevDQMpwd

dqm_interfaces.DQMcommunicator.prevDQMpwd

Definition at line 73 of file dqm_interfaces.py.

server

dqm_interfaces.DQMcommunicator.server

Definition at line 70 of file dqm_interfaces.py.

Referenced by dqm_interfaces.DQMcommunicator.cd(), dqm_interfaces.DQMcommunicator.get_root_objects(), dqm_interfaces.DQMcommunicator.get_samples(), and dqm_interfaces.DQMcommunicator.ls_url().