alcaDQMUpload Namespace Reference

Classes
class	HTTPSCertAuth
class	HTTPSCertAuthenticate
Functions
def	checkFileName
def	checkSSL
def	encode
def	filetype
def	getURL
def	main
def	marshall
def	print_help
def	registerFileAtLogServer
def	startUpload
def	upload
Variables
	HTTPS = httplib.HTTPS
	ssl_cert_file = None
	ssl_key_file = None

---

Function Documentation

def alcaDQMUpload::checkFileName

(

fileName

)

Definition at line 162 of file alcaDQMUpload.py.

                           :
    regWhitespace = re.compile('.*\s.*')
    if regWhitespace.match(fileName):
        sys.stderr.write("whitespace detected!\n")
        return False
            
    regRelval=re.compile('.*relval.*')
    regCMSSW=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_.*')
    regCMSSWpre=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_pre[0-9]+_.*')
    if regRelval.match(fileName):
        # TODO check for pre-versions
        if not regCMSSW.match(fileName):
            print "no CMSSW"
    return True
    # DQM stuff

def alcaDQMUpload::checkSSL

(

opts

)

Definition at line 116 of file alcaDQMUpload.py.

00117                   :
00118     global ssl_key_file
00119     global ssl_cert_file
00120     
00121     if opts.ssl_key_file and os.path.exists(opts.ssl_key_file):
00122         ssl_key_file = opts.ssl_key_file
00123     if opts.ssl_cert_file and os.path.exists(opts.ssl_cert_file):
00124         ssl_cert_file = opts.ssl_cert_file
00125 
00126     if not ssl_key_file:
00127         x509_path = os.getenv("X509_USER_PROXY", None)
00128         if x509_path and os.path.exists(x509_path):
00129             ssl_key_file = ssl_cert_file = x509_path
00130 
00131     if not ssl_key_file:
00132         x509_path = os.getenv("X509_USER_KEY", None)
00133         if x509_path and os.path.exists(x509_path):
00134             ssl_key_file = x509_path
00135 
00136     if not ssl_cert_file:
00137         x509_path = os.getenv("X509_USER_CERT", None)
00138         if x509_path and os.path.exists(x509_path):
00139             ssl_cert_file = x509_path
00140     
00141     if not ssl_key_file:
00142         x509_path = os.getenv("HOME") + "/.globus/userkey.pem"
00143         if os.path.exists(x509_path):
00144             ssl_key_file = x509_path
00145 
00146     if not ssl_cert_file:
00147         x509_path = os.getenv("HOME") + "/.globus/usercert.pem"
00148         if os.path.exists(x509_path):
00149             ssl_cert_file = x509_path
00150     
00151     if not ssl_key_file or not os.path.exists(ssl_key_file):
00152         sys.stderr.write("no certificate private key file found, please specify one via $X509_USER_PROXY, $X509_USER_KEY or --ssl-key-file\n")
00153         sys.exit(2)
00154                   
00155     if not ssl_cert_file or not os.path.exists(ssl_cert_file):
00156         sys.stderr.write("no certificate public key file found, please specify one via $X509_USER_CERT or --ssl-cert-file\n")
00157         sys.exit(3)
00158 
00159     print "Using SSL private key", ssl_key_file
00160     print "Using SSL public key", ssl_cert_file
00161 
                                                                                  

def alcaDQMUpload::encode	(		args,
			files
	)

  Encode form (name, value) and (name, filename, type) elements into
  multi-part/form-data. We don't actually need to know what we are
  uploading here, so just claim it's all text/plain.

Definition at line 31 of file alcaDQMUpload.py.

Referenced by trigger::TriggerEvent::addCollections(), cond::DecodingKey::createFile(), EmDQM::fillHistos(), egHLT::trigTools::getFiltersPassed(), HLTMuonIsoFilter::HLTMuonIsoFilter(), and egHLT::trigTools::setFiltersObjPasses().

                       :
  """
    Encode form (name, value) and (name, filename, type) elements into
    multi-part/form-data. We don't actually need to know what we are
    uploading here, so just claim it's all text/plain.
  """
  boundary = '----------=_DQM_FILE_BOUNDARY_=-----------'
  (body, crlf) = ('', '\r\n')
  for (key, value) in args.items():
    body += '--' + boundary + crlf
    body += ('Content-disposition: form-data; name="%s"' % key) + crlf
    body += crlf + str(value) + crlf
  for (key, filename) in files.items():
    body += '--' + boundary + crlf
    body += ('Content-Disposition: form-data; name="%s"; filename="%s"'
             % (key, os.path.basename(filename))) + crlf
    body += ('Content-Type: %s' % filetype(filename)) + crlf
    body += crlf + open(filename, "r").read() + crlf
  body += '--' + boundary + '--' + crlf + crlf
  return ('multipart/form-data; boundary=' + boundary, body)

def alcaDQMUpload::filetype

(

filename

)

Definition at line 28 of file alcaDQMUpload.py.

Referenced by SiStripQualityStatistics::analyze(), TrackerMap::save(), TrackerMap::save_as_fectrackermap(), TrackerMap::save_as_fedtrackermap(), TrackerMap::save_as_HVtrackermap(), and TrackerMap::save_as_psutrackermap().

                      :
  return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

def alcaDQMUpload::getURL	(		filename,
			destination
	)

Definition at line 200 of file alcaDQMUpload.py.

00201                                  :
00202         filename = filename.split("/")[-1]
00203         regMC=re.compile('.*_R([0-9]*)__*')
00204         if regMC.match(filename):
00205                 m = re.search('.*_R([0-9]*)(__.*).root',filename)
00206                 runNr = m.group(1)
00207                 dataset = m.group(2).replace("__","/")
00208         else:
00209                 m = re.search('.*_R([0-9]*)(_?.*).root',filename)
00210                 runNr = m.group(1)
00211                 dataset = m.group(2).replace("__","/")
00212                 if dataset=="":
00213                         dataset="/Global/Online/ALL"
00214         if not runNr:
00215                 runNr="1"
00216         if (int(runNr)==1):
00217                 return destination+"start?workspace=summary;dataset="+dataset+";sampletype=offline_data"
00218         else:
00219                 return destination+"start?workspace=summary;runnr="+runNr+";dataset="+dataset+";sampletype=online_data"
        

def alcaDQMUpload::main

(

args

)

Definition at line 237 of file alcaDQMUpload.py.

              :
    global opts
    parser = optparse.OptionParser(add_help_option=False)
    parser.add_option("-h", "--help",          action="callback", callback=print_help),
    parser.add_option("",   "--no-filename-check", dest="no_filename_check",  default=False, action="store_true")
    parser.add_option("-d", "--destination", dest="destination", default="",  action="store")
    parser.add_option("-t", "--tags", dest="tags", default="",  action="store")
    parser.add_option("-s", "--no-submission", dest="submission", default=True,  action="store_false")
    parser.add_option("-r", "--no-registration", dest="registration", default=True,  action="store_false")
    parser.add_option("","--ssl-key-file", dest="ssl_key_file", default="", action="store")
    parser.add_option("","--ssl-cert-file", dest="ssl_cert_file", default="", action="store")
    (opts, args) = parser.parse_args()
    opts.abort = False

    if not opts.destination:
        sys.stderr.write("no destination specified\n")
        sys.exit(4)
    checkSSL(opts)

    if len(args)==0:
        sys.stderr.write("no input files specified\n")
        sys.exit(1)
    for fileName in args:
        fileName=fileName.strip()
        if not os.path.exists(fileName):
            sys.stderr.write("file '%s' doesn't exist!\n" % fileName)
            continue
        if not opts.no_filename_check and not checkFileName(fileName):
            continue
        sys.stderr.write("file '%s' passed name check, upload will follow!\n" % fileName)
        if opts.submission:
          startUpload(opts.destination, fileName)
        else:
          sys.stdout.write("file '%s' would be uploaded to '%s'\n" % (fileName, opts.destination))
        if opts.registration:
          registerFileAtLogServer(fileName, opts.destination, opts.tags)       
        print "You should see the plots here: "+getURL(fileName, opts.destination)

def alcaDQMUpload::marshall	(		args,
			files,
			request
	)

  Marshalls the arguments to the CGI script as multi-part/form-data,
  not the default application/x-www-form-url-encoded.  This improves
  the transfer of the large inputs and eases command line invocation
  of the CGI script.

Definition at line 52 of file alcaDQMUpload.py.

                                  :
  """
    Marshalls the arguments to the CGI script as multi-part/form-data,
    not the default application/x-www-form-url-encoded.  This improves
    the transfer of the large inputs and eases command line invocation
    of the CGI script.
  """
  (type, body) = encode(args, files)
  request.add_header('Content-type', type)
  request.add_header('Content-length', str(len(body)))
  request.add_data(body)

def alcaDQMUpload::print_help

(

args

)

Definition at line 91 of file alcaDQMUpload.py.

                     :
    sys.stdout.write("\n")
    sys.stdout.write("This scripts intends to do the upload of files to the DQM server. It\n")
    sys.stdout.write("runs some basic checks on the file name as it is crucial to follow\n")
    sys.stdout.write("the naming convention.\n")
    sys.stdout.write("\n")
    sys.stdout.write("Mandatory Option\n")
    sys.stdout.write("    -d, --destination parameter to specify the DQM server\n")
    sys.stdout.write("\n")
    sys.stdout.write("  Proxy Options\n")
    sys.stdout.write("  The script will try to find your grid proxy automatically. To\n")
    sys.stdout.write("  do so, it checks $X509_* environment variables and your globus\n")
    sys.stdout.write("  directory (~/.globus). To override this automatism, you can use\n")
    sys.stdout.write("  the following two options:\n")
    sys.stdout.write("    --ssl-key-file          location of your private key\n")
    sys.stdout.write("    --ssl-cert-file         location of your public key\n")
    sys.stdout.write("\n")
    sys.stdout.write("Other Options\n")
    sys.stdout.write("    -h, --help                show this help message\n")
    sys.stdout.write("    -s, --no-submission       suppress the submission\n")
    sys.stdout.write("    -r, --no-registration     suppress the submission\n")
    sys.stdout.write("    --no-filename-check       omit the file name check\n")
    sys.stdout.write("\n")
    sys.exit(0)

def alcaDQMUpload::registerFileAtLogServer	(		filename,
			destination,
			tags
	)

Definition at line 220 of file alcaDQMUpload.py.

                                                        :
        filename = filename.split("/")[-1]
        regMC=re.compile('.*_R([0-9]*)__*')
        if regMC.match(filename):
                m = re.search('.*_R([0-9]*)(__.*).root',filename)
                runNr = m.group(1)
                dataset = m.group(2).replace("__","/")
        else:
                m = re.search('.*_R([0-9]*)(_?.*).root',filename)
                runNr = m.group(1)
                dataset = m.group(2).replace("__","/")
                if dataset=="":
                        dataset="/Global/Online/ALL"
        tempurl = "https://www-ekp.physik.uni-karlsruhe.de/~zeise/cgi-bin/register.py?run="+runNr+"&dataset="+dataset+"&filename="+filename+"&tags="+tags+"&instance="+destination
        print "Link that is used to register: ", tempurl
        urllib.urlopen(tempurl)

def alcaDQMUpload::startUpload	(		url,
			filename
	)

Definition at line 178 of file alcaDQMUpload.py.

                              :
    global ssl_key_file
    global ssl_cert_file

    print url, filename
    try:
        (headers, data) = \
                  upload(url,
                     { 'size': os.stat(filename).st_size,
                       'checksum': "md5:%s" % md5.new(file(filename).read()).hexdigest() },
                     { 'file': filename })
        print 'Status code: ', headers.get("Dqm-Status-Code", "None")
        print 'Message:     ', headers.get("Dqm-Status-Message", "None")
        print 'Detail:      ', headers.get("Dqm-Status-Detail", "None")
        print data
    except urllib2.HTTPError, e:
        print "ERROR", e
        print 'Status code: ', e.hdrs.get("Dqm-Status-Code", "None")
        print 'Message:     ', e.hdrs.get("Dqm-Status-Message", "None")
        print 'Detail:      ', e.hdrs.get("Dqm-Status-Detail", "None")
        sys.exit(1)

def alcaDQMUpload::upload	(		url,
			args,
			files
	)

Definition at line 64 of file alcaDQMUpload.py.

Referenced by CommissioningHistosUsingDb::doUploadAnal(), CommissioningHistosUsingDb::doUploadConf(), ApvTimingHistosUsingDb::uploadConfigurations(), LatencyHistosUsingDb::uploadConfigurations(), and FineDelayHistosUsingDb::uploadConfigurations().

                            :
  ident = "visDQMUpload DQMGUI/%s CMSSW/%s python/%s" % \
    (os.getenv('DQMGUI_VERSION', '?'),
     os.getenv('DQM_CMSSW_VERSION', os.getenv('CMSSW_VERSION', '?')),
     "%d.%d.%d" % sys.version_info[:3])
  cookie = None
  if url.startswith("https:"):
    authreq = urllib2.Request(url + '/authenticate')
    authreq.add_header('User-agent', ident)
    result = urllib2.build_opener(HTTPSCertAuthenticate()).open(authreq)
    cookie = result.headers.get('Set-Cookie')
    if not cookie:
      raise RuntimeError("Did not receive authentication cookie")
    cookie = cookie.split(";")[0]

  datareq = urllib2.Request(url + '/data/put')
  datareq.add_header('Accept-encoding', 'gzip')
  datareq.add_header('User-agent', ident)
  if cookie:
    datareq.add_header('Cookie', cookie)
  marshall(args, files, datareq)
  result = urllib2.build_opener().open(datareq)
  data = result.read()
  if result.headers.get ('Content-encoding', '') == 'gzip':
    data = gzip.GzipFile (fileobj=StringIO(data)).read ()
  return (result.headers, data)

---

Variable Documentation

alcaDQMUpload::HTTPS = httplib.HTTPS

Definition at line 13 of file alcaDQMUpload.py.

alcaDQMUpload::ssl_cert_file = None

Definition at line 18 of file alcaDQMUpload.py.

alcaDQMUpload::ssl_key_file = None

Definition at line 17 of file alcaDQMUpload.py.