fetchall_from_DQM_v2 Namespace Reference

Functions
def	auth_wget

Variables
tuple	all_files = file_list_re.findall(filedir_html)
string	base_url = 'https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/'
tuple	file_list_re = re.compile(r"<a href='[-./\w]*'>([-./\w]*)<")
list	file_res = [re.compile(r) for r in options.regexp.split(',') + [options.release]]
	Fetch the files, using multi-processing. More...
tuple	filedir_html = auth_wget(filedir_url)
string	filedir_url = base_url+relvaldir+'/'
string	help = 'Fetch data relvals.'
tuple	parser = OptionParser(usage='usage: %prog [options]')
	Define options. More...
tuple	pool = Pool(options.mthreads)
tuple	release = re.findall('(CMSSW_\d*_\d*_)\d*(?:_[\w\d]*)?', options.release)
list	releasedir = release[0]
string	relvaldir = "RelVal"
	Parse sys.argv. More...
list	selected_files = [f for f in all_files if all([r.search(f) for r in file_res])]

Function Documentation

def fetchall_from_DQM_v2.auth_wget	(		url,
			chunk_size = 1048576
	)

Returns the content of specified URL, which requires authentication.
If the content is bigger than 1MB, then save it to file.

Definition at line 38 of file fetchall_from_DQM_v2.py.

def auth_wget(url, chunk_size=1048576):
    """Returns the content of specified URL, which requires authentication.
    If the content is bigger than 1MB, then save it to file.
    """
    opener = build_opener(X509CertOpen())
    url_file = opener.open(Request(url))
    size = int(url_file.headers["Content-Length"])

    if size < 1048576:   # if File size < 1MB
        filename = basename(url)    #still download
        readed = url_file.read()    ## and then check if its not an empty dir (parent directory)
        if filename != '':
            outfile = open(filename, 'wb')  #then write File to local system
            outfile.write(readed)
        return readed

    filename = basename(url)
    file_id = selected_files.index(filename)

    if isfile("./%s" % filename):
        print '%d. Exsits on disk. Skipping.' % (file_id +1)
        return

    print '%d. Downloading...' % (file_id +1)
    file = open(filename, 'wb')
    # progress = 0
    chunk = url_file.read(chunk_size)
    while chunk:
        file.write(chunk)
        # progress += chunk_size
        chunk = url_file.read(chunk_size)
    print '%d.  Done.' % (file_id +1)
    file.close()