electronDataDiscovery Namespace Reference

Functions
def	common_search
def	getCMSdata
def	search
def	search2

Variables
tuple	dd_cond_re = re.compile(os.environ['DD_COND'].replace('*', '.*'))
tuple	dd_release_re = re.compile(os.environ['DD_RELEASE'].replace('*', '.*'))
tuple	dd_run_re = re.compile(os.environ['DD_RUN'].replace('*', '.*'))
tuple	dd_sample_re = re.compile(os.environ['DD_SAMPLE'].replace('*', '.*'))
tuple	files = dd.search()

Function Documentation

def electronDataDiscovery.common_search

(

dd_tier

)

Definition at line 60 of file electronDataDiscovery.py.

References das_client.get_data(), print(), sistrip::SpyUtilities.range(), python.rootplot.root2matplotlib.replace(), and str.

Referenced by search(), and search2().

def common_search(dd_tier):
    dd_tier_re = re.compile(dd_tier.replace('*', '.*'));

    if os.environ['DD_SOURCE'] == "das":

        query = "dataset instance=cms_dbs_prod_global"
        if os.environ['DD_RELEASE'] != "":
            query = query + " release=" + os.environ['DD_RELEASE']
        if os.environ['DD_SAMPLE'] != "":
            query = query + " primary_dataset=" + os.environ['DD_SAMPLE']
        if dd_tier != "":
            query = query + " tier=" + dd_tier
        if os.environ['DD_COND'] != "":
            query = query + " dataset=*" + os.environ['DD_COND'] + "*"
        if os.environ['DD_RUN'] != "":
            query = query + " run=" + os.environ['DD_RUN']
        # query = query + " | unique" # too long ??

        # data = os.popen('das_client.py --limit=0 --query "'+query+'"')
        # datalines = data.readlines()
        # data.close()
        # datasets = []
        # for line in datalines:
        #  line = line.rstrip()
        #  if line != "" and line[0] =="/":
        #    datasets.append(line)
        # dataset = datasets[0]

        data = das_client.json.loads(das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, 0))

        if data['nresults'] == 0:
            print('[electronDataDiscovery.py] No DAS dataset for query:', query)
            return []
        while data['nresults'] > 1:
            if data['data'][0]['dataset'][0]['name'] == data['data'][1]['dataset'][0]['name']:
                data['data'].pop(0)
                data['nresults'] -= 1
            else:
                print('[electronDataDiscovery.py] Several DAS datasets for query:', query)
                for i in range(data['nresults']):
                    print(
                        '[electronDataDiscovery.py] dataset[' + str(i) + ']: ' + data['data'][i]['dataset'][0]['name'])
                return []

        dataset = data['data'][0]['dataset'][0]['name']

        query = "file instance=cms_dbs_prod_global dataset=" + dataset

        # data = os.popen('das_client.py --limit=0 --query "'+query+'"')
        # datalines = data.readlines()
        # data.close()
        # result = []
        # for line in datalines:
        #  line = line.rstrip()
        #  if line != "" and line[0] =="/":
        #    result.append(line)

        data = das_client.json.loads(das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, 0))

        if data['nresults'] == 0:
            print('[electronDataDiscovery.py] No DAS file in dataset:', dataset)
            return []
        else:
            print('there is %d results' % nresults)

        result = []
        for i in range(0, data['nresults']):
            result.append(str(data['data'][i]['file'][0]['name']))

    elif os.environ['DD_SOURCE'] == "dbs":

        input = "find file"
        separator = " where "
        if os.environ['DD_RELEASE'] != "":
            input = input + separator + "release = " + os.environ['DD_RELEASE']
            separator = " and "
        if os.environ['DD_SAMPLE'] != "":
            input = input + separator + "primds = " + os.environ['DD_SAMPLE']
            separator = " and "
        if os.environ['DD_RUN'] != "":
            input = input + separator + "run = " + os.environ['DD_RUN']
            separator = " and "
        input = input + separator + "dataset like *" + os.environ['DD_COND'] + "*" + dd_tier + "*"

        data = os.popen(
            'dbs search --url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" --query "' + input + '"')
        datalines = data.readlines()
        data.close()
        result = []
        for line in datalines:
            line = line.rstrip()
            if line != "" and line[0] == "/":
                result.append(line)

    elif os.environ['DD_SOURCE'] == "http":

        input = "find file"
        separator = " where "
        if os.environ['DD_RELEASE'] != "":
            input = input + separator + "release = " + os.environ['DD_RELEASE']
            separator = " and "
        if os.environ['DD_SAMPLE'] != "":
            input = input + separator + "primds = " + os.environ['DD_SAMPLE']
            separator = " and "
        if os.environ['DD_RUN'] != "":
            input = input + separator + "run = " + os.environ['DD_RUN']
            separator = " and "
        input = input + separator + "dataset like *" + os.environ['DD_COND'] + "*" + dd_tier + "*"

        url = "https://cmsweb.cern.ch:443/dbs_discovery/aSearch"
        final_input = urllib.quote(input);

        agent = "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"
        ctypes = "text/plain"
        headers = {'User-Agent': agent, 'Accept': ctypes}
        params = {'dbsInst': 'cms_dbs_prod_global',
                  'html': 0, 'caseSensitive': 'on', '_idx': 0, 'pagerStep': -1,
                  'userInput': final_input,
                  'xml': 0, 'details': 0, 'cff': 0, 'method': 'dbsapi'}
        data = urllib.urlencode(params, doseq=True)
        req = urllib2.Request(url, data, headers)
        data = ""

        try:
            response = urllib2.urlopen(req)
            data = response.read()
        except urllib2.HTTPError as e:
            if e.code == 201:
                print(e.headers)
                print(e.msg)
                pass
            else:
                raise e

        datalines = data.readlines()
        data.close()
        result = []
        for line in datalines:
            line = line.rstrip()
            if line != "" and line[0] == "/":
                result.append(line)

    elif os.environ['DD_SOURCE'] == "lsf":

        dbs_path = '/' + os.environ['DD_SAMPLE'] + '/' + os.environ['DD_RELEASE'] + '-' + os.environ['DD_COND'] + '/' + \
                   os.environ['DD_TIER'] + '"'
        if __name__ == "__main__":
            print('dbs path:', dbs_path)
        data = os.popen('dbs lsf --path="' + dbs_path + '"')
        datalines = data.readlines()
        data.close()
        result = []
        for line in datalines:
            line = line.rstrip()
            if line != "" and line[0] == "/":
                result.append(line)

    elif os.environ['DD_SOURCE'].startswith('/castor/cern.ch/cms/'):  # assumed to be a castor dir

        castor_dir = os.environ['DD_SOURCE'].replace('/castor/cern.ch/cms/', '/', 1)
        result = []
        data = os.popen('rfdir /castor/cern.ch/cms' + castor_dir)
        subdirs = data.readlines()
        data.close()
        datalines = []
        for line in subdirs:
            line = line.rstrip()
            subdir = line.split()[8]
            data = os.popen('rfdir /castor/cern.ch/cms' + castor_dir + '/' + subdir)
            datalines = data.readlines()
            for line in datalines:
                line = line.rstrip()
                file = line.split()[8]
                if file != "":
                    result.append(castor_dir + '/' + subdir + '/' + file)
            data.close()

    elif os.environ['DD_SOURCE'].startswith('/eos/cms/'):  # assumed to be an eos dir

        data = os.popen('eos find -f ' + os.environ['DD_SOURCE'])
        lines = data.readlines()
        data.close()
        result = []
        for line in lines:
            line = line.strip().replace('/eos/cms/', '/', 1)
            if line == "": continue
            if dd_sample_re.search(line) == None: continue
            if dd_cond_re.search(line) == None: continue
            if dd_tier_re.search(line) == None: continue
            if dd_run_re.search(line) == None: continue
            result.append(line)

    else:  # os.environ['DD_SOURCE'] is assumed to be a file name

        result = []
        for line in open(os.environ['DD_SOURCE']).readlines():
            line = os.path.expandvars(line.strip())
            if line == "": continue
            if dd_sample_re.search(line) == None: continue
            if dd_cond_re.search(line) == None: continue
            if dd_tier_re.search(line) == None: continue
            if dd_run_re.search(line) == None: continue
            result.append(line)

        if len(result) == 0:
            diag = '[electronDataDiscovery.py] No more files after filtering with :'
            if os.environ['DD_SAMPLE'] != '': diag += ' ' + os.environ['DD_SAMPLE']
            if os.environ['DD_COND'] != '': diag += ' ' + os.environ['DD_COND']
            if dd_tier != '': diag += ' ' + dd_tier
            if os.environ['DD_RUN'] != '': diag += ' ' + os.environ['DD_RUN']
            print(diag)

    return result