CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
alcaDQMUpload Namespace Reference

Classes

class  HTTPSCertAuth
 
class  HTTPSCertAuthenticate
 

Functions

def checkFileName
 
def checkSSL
 
def encode
 
def filetype
 
def getURL
 
def main
 
def marshall
 
def print_help
 
def registerFileAtLogServer
 
def startUpload
 
def upload
 

Variables

 HTTPS = httplib.HTTPS
 
 ssl_cert_file = None
 
 ssl_key_file = None
 

Function Documentation

def alcaDQMUpload.checkFileName (   fileName)

Definition at line 162 of file alcaDQMUpload.py.

Referenced by main().

162 
163 def checkFileName(fileName):
164  regWhitespace = re.compile('.*\s.*')
165  if regWhitespace.match(fileName):
166  sys.stderr.write("whitespace detected!\n")
167  return False
168 
169  regRelval=re.compile('.*relval.*')
170  regCMSSW=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_.*')
171  regCMSSWpre=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_pre[0-9]+_.*')
172  if regRelval.match(fileName):
173  # TODO check for pre-versions
174  if not regCMSSW.match(fileName):
175  print "no CMSSW"
176  return True
177  # DQM stuff
def alcaDQMUpload.checkSSL (   opts)

Definition at line 116 of file alcaDQMUpload.py.

Referenced by main().

116 
117 def checkSSL(opts):
118  global ssl_key_file
119  global ssl_cert_file
120 
121  if opts.ssl_key_file and os.path.exists(opts.ssl_key_file):
122  ssl_key_file = opts.ssl_key_file
123  if opts.ssl_cert_file and os.path.exists(opts.ssl_cert_file):
124  ssl_cert_file = opts.ssl_cert_file
125 
126  if not ssl_key_file:
127  x509_path = os.getenv("X509_USER_PROXY", None)
128  if x509_path and os.path.exists(x509_path):
129  ssl_key_file = ssl_cert_file = x509_path
130 
131  if not ssl_key_file:
132  x509_path = os.getenv("X509_USER_KEY", None)
133  if x509_path and os.path.exists(x509_path):
134  ssl_key_file = x509_path
135 
136  if not ssl_cert_file:
137  x509_path = os.getenv("X509_USER_CERT", None)
138  if x509_path and os.path.exists(x509_path):
139  ssl_cert_file = x509_path
140 
141  if not ssl_key_file:
142  x509_path = os.getenv("HOME") + "/.globus/userkey.pem"
143  if os.path.exists(x509_path):
144  ssl_key_file = x509_path
145 
146  if not ssl_cert_file:
147  x509_path = os.getenv("HOME") + "/.globus/usercert.pem"
148  if os.path.exists(x509_path):
149  ssl_cert_file = x509_path
150 
151  if not ssl_key_file or not os.path.exists(ssl_key_file):
152  sys.stderr.write("no certificate private key file found, please specify one via $X509_USER_PROXY, $X509_USER_KEY or --ssl-key-file\n")
153  sys.exit(2)
154 
155  if not ssl_cert_file or not os.path.exists(ssl_cert_file):
156  sys.stderr.write("no certificate public key file found, please specify one via $X509_USER_CERT or --ssl-cert-file\n")
157  sys.exit(3)
158 
159  print "Using SSL private key", ssl_key_file
160  print "Using SSL public key", ssl_cert_file
161 
def alcaDQMUpload.encode (   args,
  files 
) 
  Encode form (name, value) and (name, filename, type) elements into
  multi-part/form-data. We don't actually need to know what we are
  uploading here, so just claim it's all text/plain.

Definition at line 31 of file alcaDQMUpload.py.

References filetype().

Referenced by BeautifulSoup.UnicodeDammit._detectEncoding(), BeautifulSoup.Tag._invert(), BeautifulSoup.UnicodeDammit._subMSChar(), trigger::TriggerEvent.addCollections(), EmDQM.fillHistos(), egHLT::trigTools.fillHLTposition(), egHLT::trigTools.getFiltersPassed(), HLTMuonIsoFilter.HLTMuonIsoFilter(), marshall(), and egHLT::trigTools.setFiltersObjPasses().

31 
32 def encode(args, files):
33  """
34  Encode form (name, value) and (name, filename, type) elements into
35  multi-part/form-data. We don't actually need to know what we are
36  uploading here, so just claim it's all text/plain.
37  """
38  boundary = '----------=_DQM_FILE_BOUNDARY_=-----------'
39  (body, crlf) = ('', '\r\n')
40  for (key, value) in args.items():
41  body += '--' + boundary + crlf
42  body += ('Content-disposition: form-data; name="%s"' % key) + crlf
43  body += crlf + str(value) + crlf
44  for (key, filename) in files.items():
45  body += '--' + boundary + crlf
46  body += ('Content-Disposition: form-data; name="%s"; filename="%s"'
47  % (key, os.path.basename(filename))) + crlf
48  body += ('Content-Type: %s' % filetype(filename)) + crlf
49  body += crlf + open(filename, "r").read() + crlf
50  body += '--' + boundary + '--' + crlf + crlf
51  return ('multipart/form-data; boundary=' + boundary, body)
def alcaDQMUpload.filetype (   filename)

Definition at line 28 of file alcaDQMUpload.py.

Referenced by SiStripQualityStatistics.analyze(), encode(), TrackerMap.printall(), TrackerMap.save(), TrackerMap.save_as_fectrackermap(), TrackerMap.save_as_fedtrackermap(), TrackerMap.save_as_HVtrackermap(), and TrackerMap.save_as_psutrackermap().

28 
29 def filetype(filename):
30  return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
def alcaDQMUpload.getURL (   filename,
  destination 
)

Definition at line 200 of file alcaDQMUpload.py.

References linker.replace().

Referenced by main().

200 
201 def getURL(filename, destination):
202  filename = filename.split("/")[-1]
203  regMC=re.compile('.*_R([0-9]*)__*')
204  if regMC.match(filename):
205  m = re.search('.*_R([0-9]*)(__.*).root',filename)
206  runNr = m.group(1)
207  dataset = m.group(2).replace("__","/")
208  else:
209  m = re.search('.*_R([0-9]*)(_?.*).root',filename)
210  runNr = m.group(1)
211  dataset = m.group(2).replace("__","/")
212  if dataset=="":
213  dataset="/Global/Online/ALL"
214  if not runNr:
215  runNr="1"
216  if (int(runNr)==1):
217  return destination+"start?workspace=summary;dataset="+dataset+";sampletype=offline_data"
218  else:
219  return destination+"start?workspace=summary;runnr="+runNr+";dataset="+dataset+";sampletype=online_data"
linker.replace
def replace
Definition: linker.py:10
def alcaDQMUpload.main (   args)

Definition at line 237 of file alcaDQMUpload.py.

References checkFileName(), checkSSL(), getURL(), registerFileAtLogServer(), and startUpload().

237 
238 def main(args):
239  global opts
240  parser = optparse.OptionParser(add_help_option=False)
241  parser.add_option("-h", "--help", action="callback", callback=print_help),
242  parser.add_option("", "--no-filename-check", dest="no_filename_check", default=False, action="store_true")
243  parser.add_option("-d", "--destination", dest="destination", default="", action="store")
244  parser.add_option("-t", "--tags", dest="tags", default="", action="store")
245  parser.add_option("-s", "--no-submission", dest="submission", default=True, action="store_false")
246  parser.add_option("-r", "--no-registration", dest="registration", default=True, action="store_false")
247  parser.add_option("","--ssl-key-file", dest="ssl_key_file", default="", action="store")
248  parser.add_option("","--ssl-cert-file", dest="ssl_cert_file", default="", action="store")
249  (opts, args) = parser.parse_args()
250  opts.abort = False
251 
252  if not opts.destination:
253  sys.stderr.write("no destination specified\n")
254  sys.exit(4)
255  checkSSL(opts)
256 
257  if len(args)==0:
258  sys.stderr.write("no input files specified\n")
259  sys.exit(1)
260  for fileName in args:
261  fileName=fileName.strip()
262  if not os.path.exists(fileName):
263  sys.stderr.write("file '%s' doesn't exist!\n" % fileName)
264  continue
265  if not opts.no_filename_check and not checkFileName(fileName):
266  continue
267  sys.stderr.write("file '%s' passed name check, upload will follow!\n" % fileName)
268  if opts.submission:
269  startUpload(opts.destination, fileName)
270  else:
271  sys.stdout.write("file '%s' would be uploaded to '%s'\n" % (fileName, opts.destination))
272  if opts.registration:
273  registerFileAtLogServer(fileName, opts.destination, opts.tags)
274  print "You should see the plots here: "+getURL(fileName, opts.destination)
alcaDQMUpload.registerFileAtLogServer
def registerFileAtLogServer
Definition: alcaDQMUpload.py:220
def alcaDQMUpload.marshall (   args,
  files,
  request 
) 
  Marshalls the arguments to the CGI script as multi-part/form-data,
  not the default application/x-www-form-url-encoded.  This improves
  the transfer of the large inputs and eases command line invocation
  of the CGI script.

Definition at line 52 of file alcaDQMUpload.py.

References encode().

Referenced by upload().

52 
53 def marshall(args, files, request):
54  """
55  Marshalls the arguments to the CGI script as multi-part/form-data,
56  not the default application/x-www-form-url-encoded. This improves
57  the transfer of the large inputs and eases command line invocation
58  of the CGI script.
59  """
60  (type, body) = encode(args, files)
61  request.add_header('Content-type', type)
62  request.add_header('Content-length', str(len(body)))
63  request.add_data(body)
def alcaDQMUpload.print_help (   args)

Definition at line 91 of file alcaDQMUpload.py.

91 
92 def print_help(*args):
93  sys.stdout.write("\n")
94  sys.stdout.write("This scripts intends to do the upload of files to the DQM server. It\n")
95  sys.stdout.write("runs some basic checks on the file name as it is crucial to follow\n")
96  sys.stdout.write("the naming convention.\n")
97  sys.stdout.write("\n")
98  sys.stdout.write("Mandatory Option\n")
99  sys.stdout.write(" -d, --destination parameter to specify the DQM server\n")
100  sys.stdout.write("\n")
101  sys.stdout.write(" Proxy Options\n")
102  sys.stdout.write(" The script will try to find your grid proxy automatically. To\n")
103  sys.stdout.write(" do so, it checks $X509_* environment variables and your globus\n")
104  sys.stdout.write(" directory (~/.globus). To override this automatism, you can use\n")
105  sys.stdout.write(" the following two options:\n")
106  sys.stdout.write(" --ssl-key-file location of your private key\n")
107  sys.stdout.write(" --ssl-cert-file location of your public key\n")
108  sys.stdout.write("\n")
109  sys.stdout.write("Other Options\n")
110  sys.stdout.write(" -h, --help show this help message\n")
111  sys.stdout.write(" -s, --no-submission suppress the submission\n")
112  sys.stdout.write(" -r, --no-registration suppress the submission\n")
113  sys.stdout.write(" --no-filename-check omit the file name check\n")
114  sys.stdout.write("\n")
115  sys.exit(0)
def alcaDQMUpload.registerFileAtLogServer (   filename,
  destination,
  tags 
)

Definition at line 220 of file alcaDQMUpload.py.

References linker.replace().

Referenced by main().

220 
221 def registerFileAtLogServer(filename, destination, tags):
222  filename = filename.split("/")[-1]
223  regMC=re.compile('.*_R([0-9]*)__*')
224  if regMC.match(filename):
225  m = re.search('.*_R([0-9]*)(__.*).root',filename)
226  runNr = m.group(1)
227  dataset = m.group(2).replace("__","/")
228  else:
229  m = re.search('.*_R([0-9]*)(_?.*).root',filename)
230  runNr = m.group(1)
231  dataset = m.group(2).replace("__","/")
232  if dataset=="":
233  dataset="/Global/Online/ALL"
234  tempurl = "https://www-ekp.physik.uni-karlsruhe.de/~zeise/cgi-bin/register.py?run="+runNr+"&dataset="+dataset+"&filename="+filename+"&tags="+tags+"&instance="+destination
235  print "Link that is used to register: ", tempurl
236  urllib.urlopen(tempurl)
linker.replace
def replace
Definition: linker.py:10
alcaDQMUpload.registerFileAtLogServer
def registerFileAtLogServer
Definition: alcaDQMUpload.py:220
def alcaDQMUpload.startUpload (   url,
  filename 
)

Definition at line 178 of file alcaDQMUpload.py.

References mergeVDriftHistosByStation.file, SiPixelLorentzAngle_cfi.read, and upload().

Referenced by main().

178 
179 def startUpload(url, filename):
180  global ssl_key_file
181  global ssl_cert_file
182 
183  print url, filename
184  try:
185  (headers, data) = \
186  upload(url,
187  { 'size': os.stat(filename).st_size,
188  'checksum': "md5:%s" % md5.new(file(filename).read()).hexdigest() },
189  { 'file': filename })
190  print 'Status code: ', headers.get("Dqm-Status-Code", "None")
191  print 'Message: ', headers.get("Dqm-Status-Message", "None")
192  print 'Detail: ', headers.get("Dqm-Status-Detail", "None")
193  print data
194  except urllib2.HTTPError, e:
195  print "ERROR", e
196  print 'Status code: ', e.hdrs.get("Dqm-Status-Code", "None")
197  print 'Message: ', e.hdrs.get("Dqm-Status-Message", "None")
198  print 'Detail: ', e.hdrs.get("Dqm-Status-Detail", "None")
199  sys.exit(1)
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7
def alcaDQMUpload.upload (   url,
  args,
  files 
)

Definition at line 64 of file alcaDQMUpload.py.

References marshall().

Referenced by CommissioningHistosUsingDb.doUploadAnal(), CommissioningHistosUsingDb.doUploadConf(), startUpload(), ApvTimingHistosUsingDb.uploadConfigurations(), LatencyHistosUsingDb.uploadConfigurations(), and FineDelayHistosUsingDb.uploadConfigurations().

64 
65 def upload(url, args, files):
66  ident = "visDQMUpload DQMGUI/%s CMSSW/%s python/%s" % \
67  (os.getenv('DQMGUI_VERSION', '?'),
68  os.getenv('DQM_CMSSW_VERSION', os.getenv('CMSSW_VERSION', '?')),
69  "%d.%d.%d" % sys.version_info[:3])
70  cookie = None
71  if url.startswith("https:"):
72  authreq = urllib2.Request(url + '/authenticate')
73  authreq.add_header('User-agent', ident)
74  result = urllib2.build_opener(HTTPSCertAuthenticate()).open(authreq)
75  cookie = result.headers.get('Set-Cookie')
76  if not cookie:
77  raise RuntimeError("Did not receive authentication cookie")
78  cookie = cookie.split(";")[0]
79 
80  datareq = urllib2.Request(url + '/data/put')
81  datareq.add_header('Accept-encoding', 'gzip')
82  datareq.add_header('User-agent', ident)
83  if cookie:
84  datareq.add_header('Cookie', cookie)
85  marshall(args, files, datareq)
86  result = urllib2.build_opener().open(datareq)
87  data = result.read()
88  if result.headers.get ('Content-encoding', '') == 'gzip':
89  data = gzip.GzipFile (fileobj=StringIO(data)).read ()
90  return (result.headers, data)

Variable Documentation

alcaDQMUpload.HTTPS = httplib.HTTPS

Definition at line 13 of file alcaDQMUpload.py.

alcaDQMUpload.ssl_cert_file = None

Definition at line 18 of file alcaDQMUpload.py.

alcaDQMUpload.ssl_key_file = None

Definition at line 17 of file alcaDQMUpload.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5