00001
00002
00003 import sys, os, os.path, re, string, httpslib, mimetypes, urllib, urllib2, httpslib, gzip, md5
00004 from cStringIO import StringIO
00005 from stat import *
00006 import optparse
00007
00008 try:
00009 import hashlib
00010 except:
00011 pass
00012
00013 HTTPS = httpslib.HTTPS
00014 if sys.version_info[:3] >= (2, 4, 0):
00015 HTTPS = httpslib.HTTPSConnection
00016
00017 ssl_key_file = None
00018 ssl_cert_file = None
00019
00020 class HTTPSCertAuth(HTTPS):
00021 def __init__(self, host, *args, **kwargs):
00022 HTTPS.__init__(self, host, key_file = ssl_key_file, cert_file = ssl_cert_file, **kwargs)
00023
00024 class HTTPSCertAuthenticate(urllib2.AbstractHTTPHandler):
00025 def default_open(self, req):
00026 return self.do_open(HTTPSCertAuth, req)
00027
00028 def filetype(filename):
00029 return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
00030
00031 def encode(args, files):
00032 """
00033 Encode form (name, value) and (name, filename, type) elements into
00034 multi-part/form-data. We don't actually need to know what we are
00035 uploading here, so just claim it's all text/plain.
00036 """
00037 boundary = '----------=_DQM_FILE_BOUNDARY_=-----------'
00038 (body, crlf) = ('', '\r\n')
00039 for (key, value) in args.items():
00040 body += '--' + boundary + crlf
00041 body += ('Content-disposition: form-data; name="%s"' % key) + crlf
00042 body += crlf + str(value) + crlf
00043 for (key, filename) in files.items():
00044 body += '--' + boundary + crlf
00045 body += ('Content-Disposition: form-data; name="%s"; filename="%s"'
00046 % (key, os.path.basename(filename))) + crlf
00047 body += ('Content-Type: %s' % filetype(filename)) + crlf
00048 body += crlf + open(filename, "r").read() + crlf
00049 body += '--' + boundary + '--' + crlf + crlf
00050 return ('multipart/form-data; boundary=' + boundary, body)
00051
00052 def marshall(args, files, request):
00053 """
00054 Marshalls the arguments to the CGI script as multi-part/form-data,
00055 not the default application/x-www-form-url-encoded. This improves
00056 the transfer of the large inputs and eases command line invocation
00057 of the CGI script.
00058 """
00059 (type, body) = encode(args, files)
00060 request.add_header('Content-type', type)
00061 request.add_header('Content-length', str(len(body)))
00062 request.add_data(body)
00063
00064 def upload(url, args, files):
00065 ident = "visDQMUpload DQMGUI/%s CMSSW/%s python/%s" % \
00066 (os.getenv('DQMGUI_VERSION', '?'),
00067 os.getenv('DQM_CMSSW_VERSION', os.getenv('CMSSW_VERSION', '?')),
00068 "%d.%d.%d" % sys.version_info[:3])
00069 cookie = None
00070 if url.startswith("https:"):
00071 authreq = urllib2.Request(url + '/authenticate')
00072 authreq.add_header('User-agent', ident)
00073 result = urllib2.build_opener(HTTPSCertAuthenticate()).open(authreq)
00074 cookie = result.headers.get('Set-Cookie')
00075 if not cookie:
00076 raise RuntimeError("Did not receive authentication cookie")
00077 cookie = cookie.split(";")[0]
00078
00079 datareq = urllib2.Request(url + '/data/put')
00080 datareq.add_header('Accept-encoding', 'gzip')
00081 datareq.add_header('User-agent', ident)
00082 if cookie:
00083 datareq.add_header('Cookie', cookie)
00084 marshall(args, files, datareq)
00085 result = urllib2.build_opener().open(datareq)
00086 data = result.read()
00087 if result.headers.get ('Content-encoding', '') == 'gzip':
00088 data = gzip.GzipFile (fileobj=StringIO(data)).read ()
00089 return (result.headers, data)
00090
00091 def print_help(*args):
00092 sys.stdout.write("\n")
00093 sys.stdout.write("This scripts intends to do the upload of files to the DQM server. It\n")
00094 sys.stdout.write("runs some basic checks on the file name as it is crucial to follow\n")
00095 sys.stdout.write("the naming convention.\n")
00096 sys.stdout.write("\n")
00097 sys.stdout.write("Mandatory Option\n")
00098 sys.stdout.write(" -d, --destination parameter to specify the DQM server\n")
00099 sys.stdout.write("\n")
00100 sys.stdout.write(" Proxy Options\n")
00101 sys.stdout.write(" The script will try to find your grid proxy automatically. To\n")
00102 sys.stdout.write(" do so, it checks $X509_* environment variables and your globus\n")
00103 sys.stdout.write(" directory (~/.globus). To override this automatism, you can use\n")
00104 sys.stdout.write(" the following two options:\n")
00105 sys.stdout.write(" --ssl-key-file location of your private key\n")
00106 sys.stdout.write(" --ssl-cert-file location of your public key\n")
00107 sys.stdout.write("\n")
00108 sys.stdout.write("Other Options\n")
00109 sys.stdout.write(" -h, --help show this help message\n")
00110 sys.stdout.write(" -s, --no-submission suppress the submission\n")
00111 sys.stdout.write(" -r, --no-registration suppress the submission\n")
00112 sys.stdout.write(" --no-filename-check omit the file name check\n")
00113 sys.stdout.write("\n")
00114 sys.exit(0)
00115
00116 def checkSSL(opts):
00117 global ssl_key_file
00118 global ssl_cert_file
00119
00120 if opts.ssl_key_file and os.path.exists(opts.ssl_key_file):
00121 ssl_key_file = opts.ssl_key_file
00122 if opts.ssl_cert_file and os.path.exists(opts.ssl_cert_file):
00123 ssl_cert_file = opts.ssl_cert_file
00124
00125 if not ssl_key_file:
00126 x509_path = os.getenv("X509_USER_PROXY", None)
00127 if x509_path and os.path.exists(x509_path):
00128 ssl_key_file = ssl_cert_file = x509_path
00129
00130 if not ssl_key_file:
00131 x509_path = os.getenv("X509_USER_KEY", None)
00132 if x509_path and os.path.exists(x509_path):
00133 ssl_key_file = x509_path
00134
00135 if not ssl_cert_file:
00136 x509_path = os.getenv("X509_USER_CERT", None)
00137 if x509_path and os.path.exists(x509_path):
00138 ssl_cert_file = x509_path
00139
00140 if not ssl_key_file:
00141 x509_path = os.getenv("HOME") + "/.globus/userkey.pem"
00142 if os.path.exists(x509_path):
00143 ssl_key_file = x509_path
00144
00145 if not ssl_cert_file:
00146 x509_path = os.getenv("HOME") + "/.globus/usercert.pem"
00147 if os.path.exists(x509_path):
00148 ssl_cert_file = x509_path
00149
00150 if not ssl_key_file or not os.path.exists(ssl_key_file):
00151 sys.stderr.write("no certificate private key file found, please specify one via $X509_USER_PROXY, $X509_USER_KEY or --ssl-key-file\n")
00152 sys.exit(2)
00153
00154 if not ssl_cert_file or not os.path.exists(ssl_cert_file):
00155 sys.stderr.write("no certificate public key file found, please specify one via $X509_USER_CERT or --ssl-cert-file\n")
00156 sys.exit(3)
00157
00158 print "Using SSL private key", ssl_key_file
00159 print "Using SSL public key", ssl_cert_file
00160
00161
00162 def checkFileName(fileName):
00163 regWhitespace = re.compile('.*\s.*')
00164 if regWhitespace.match(fileName):
00165 sys.stderr.write("whitespace detected!\n")
00166 return False
00167
00168 regRelval=re.compile('.*relval.*')
00169 regCMSSW=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_.*')
00170 regCMSSWpre=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_pre[0-9]+_.*')
00171 if regRelval.match(fileName):
00172
00173 if not regCMSSW.match(fileName):
00174 print "no CMSSW"
00175 return True
00176
00177
00178 def startUpload(url, filename):
00179 global ssl_key_file
00180 global ssl_cert_file
00181
00182 print url, filename
00183 try:
00184 (headers, data) = \
00185 upload(url,
00186 { 'size': os.stat(filename).st_size,
00187 'checksum': "md5:%s" % md5.new(file(filename).read()).hexdigest() },
00188 { 'file': filename })
00189 print 'Status code: ', headers.get("Dqm-Status-Code", "None")
00190 print 'Message: ', headers.get("Dqm-Status-Message", "None")
00191 print 'Detail: ', headers.get("Dqm-Status-Detail", "None")
00192 print data
00193 except urllib2.HTTPError, e:
00194 print "ERROR", e
00195 print 'Status code: ', e.hdrs.get("Dqm-Status-Code", "None")
00196 print 'Message: ', e.hdrs.get("Dqm-Status-Message", "None")
00197 print 'Detail: ', e.hdrs.get("Dqm-Status-Detail", "None")
00198 sys.exit(1)
00199
00200 def getURL(filename, destination):
00201 filename = filename.split("/")[-1]
00202 regMC=re.compile('.*_R([0-9]*)__*')
00203 if regMC.match(filename):
00204 m = re.search('.*_R([0-9]*)(__.*).root',filename)
00205 runNr = m.group(1)
00206 dataset = m.group(2).replace("__","/")
00207 else:
00208 m = re.search('.*_R([0-9]*)(_?.*).root',filename)
00209 runNr = m.group(1)
00210 dataset = m.group(2).replace("__","/")
00211 if dataset=="":
00212 dataset="/Global/Online/ALL"
00213 if not runNr:
00214 runNr="1"
00215 if (int(runNr)==1):
00216 return destination+"start?workspace=summary;dataset="+dataset+";sampletype=offline_data"
00217 else:
00218 return destination+"start?workspace=summary;runnr="+runNr+";dataset="+dataset+";sampletype=online_data"
00219
00220 def registerFileAtLogServer(filename, destination, tags):
00221 filename = filename.split("/")[-1]
00222 regMC=re.compile('.*_R([0-9]*)__*')
00223 if regMC.match(filename):
00224 m = re.search('.*_R([0-9]*)(__.*).root',filename)
00225 runNr = m.group(1)
00226 dataset = m.group(2).replace("__","/")
00227 else:
00228 m = re.search('.*_R([0-9]*)(_?.*).root',filename)
00229 runNr = m.group(1)
00230 dataset = m.group(2).replace("__","/")
00231 if dataset=="":
00232 dataset="/Global/Online/ALL"
00233 tempurl = "https://www-ekp.physik.uni-karlsruhe.de/~zeise/cgi-bin/register.py?run="+runNr+"&dataset="+dataset+"&filename="+filename+"&tags="+tags+"&instance="+destination
00234 print "Link that is used to register: ", tempurl
00235 urllib.urlopen(tempurl)
00236
00237 def main(args):
00238 global opts
00239 parser = optparse.OptionParser(add_help_option=False)
00240 parser.add_option("-h", "--help", action="callback", callback=print_help),
00241 parser.add_option("", "--no-filename-check", dest="no_filename_check", default=False, action="store_true")
00242 parser.add_option("-d", "--destination", dest="destination", default="", action="store")
00243 parser.add_option("-t", "--tags", dest="tags", default="", action="store")
00244 parser.add_option("-s", "--no-submission", dest="submission", default=True, action="store_false")
00245 parser.add_option("-r", "--no-registration", dest="registration", default=True, action="store_false")
00246 parser.add_option("","--ssl-key-file", dest="ssl_key_file", default="", action="store")
00247 parser.add_option("","--ssl-cert-file", dest="ssl_cert_file", default="", action="store")
00248 (opts, args) = parser.parse_args()
00249 opts.abort = False
00250
00251 if not opts.destination:
00252 sys.stderr.write("no destination specified\n")
00253 sys.exit(4)
00254 checkSSL(opts)
00255
00256 if len(args)==0:
00257 sys.stderr.write("no input files specified\n")
00258 sys.exit(1)
00259 for fileName in args:
00260 fileName=fileName.strip()
00261 if not os.path.exists(fileName):
00262 sys.stderr.write("file '%s' doesn't exist!\n" % fileName)
00263 continue
00264 if not opts.no_filename_check and not checkFileName(fileName):
00265 continue
00266 sys.stderr.write("file '%s' passed name check, upload will follow!\n" % fileName)
00267 if opts.submission:
00268 startUpload(opts.destination, fileName)
00269 else:
00270 sys.stdout.write("file '%s' would be uploaded to '%s'\n" % (fileName, opts.destination))
00271 if opts.registration:
00272 registerFileAtLogServer(fileName, opts.destination, opts.tags)
00273 print "You should see the plots here: "+getURL(fileName, opts.destination)
00274
00275 if __name__ == '__main__':
00276 sys.exit(main(sys.argv[1:]))
00277
00278
1.7.3