CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Classes | Functions | Variables
alcaDQMUpload Namespace Reference

Classes

class  HTTPSCertAuth
 
class  HTTPSCertAuthenticate
 

Functions

def checkFileName
 
def checkSSL
 
def encode
 
def filetype
 
def getURL
 
def main
 
def marshall
 
def print_help
 
def registerFileAtLogServer
 
def startUpload
 
def upload
 

Variables

 HTTPS = httplib.HTTPS
 
 ssl_cert_file = None
 
 ssl_key_file = None
 

Function Documentation

def alcaDQMUpload.checkFileName (   fileName)

Definition at line 163 of file alcaDQMUpload.py.

References print().

Referenced by main().

163 
164 def checkFileName(fileName):
165  regWhitespace = re.compile('.*\s.*')
166  if regWhitespace.match(fileName):
167  sys.stderr.write("whitespace detected!\n")
168  return False
169 
170  regRelval=re.compile('.*relval.*')
171  regCMSSW=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_.*')
172  regCMSSWpre=re.compile('.*CMSSW_[0-9]+_[0-9]+_[0-9]+_pre[0-9]+_.*')
173  if regRelval.match(fileName):
174  # TODO check for pre-versions
175  if not regCMSSW.match(fileName):
176  print("no CMSSW")
177  return True
178  # DQM stuff
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def alcaDQMUpload.checkSSL (   opts)

Definition at line 117 of file alcaDQMUpload.py.

References print().

Referenced by main().

117 
118 def checkSSL(opts):
119  global ssl_key_file
120  global ssl_cert_file
121 
122  if opts.ssl_key_file and os.path.exists(opts.ssl_key_file):
123  ssl_key_file = opts.ssl_key_file
124  if opts.ssl_cert_file and os.path.exists(opts.ssl_cert_file):
125  ssl_cert_file = opts.ssl_cert_file
126 
127  if not ssl_key_file:
128  x509_path = os.getenv("X509_USER_PROXY", None)
129  if x509_path and os.path.exists(x509_path):
130  ssl_key_file = ssl_cert_file = x509_path
131 
132  if not ssl_key_file:
133  x509_path = os.getenv("X509_USER_KEY", None)
134  if x509_path and os.path.exists(x509_path):
135  ssl_key_file = x509_path
136 
137  if not ssl_cert_file:
138  x509_path = os.getenv("X509_USER_CERT", None)
139  if x509_path and os.path.exists(x509_path):
140  ssl_cert_file = x509_path
141 
142  if not ssl_key_file:
143  x509_path = os.getenv("HOME") + "/.globus/userkey.pem"
144  if os.path.exists(x509_path):
145  ssl_key_file = x509_path
146 
147  if not ssl_cert_file:
148  x509_path = os.getenv("HOME") + "/.globus/usercert.pem"
149  if os.path.exists(x509_path):
150  ssl_cert_file = x509_path
151 
152  if not ssl_key_file or not os.path.exists(ssl_key_file):
153  sys.stderr.write("no certificate private key file found, please specify one via $X509_USER_PROXY, $X509_USER_KEY or --ssl-key-file\n")
154  sys.exit(2)
155 
156  if not ssl_cert_file or not os.path.exists(ssl_cert_file):
157  sys.stderr.write("no certificate public key file found, please specify one via $X509_USER_CERT or --ssl-cert-file\n")
158  sys.exit(3)
159 
160  print("Using SSL private key", ssl_key_file)
161  print("Using SSL public key", ssl_cert_file)
162 
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def alcaDQMUpload.encode (   args,
  files 
) 
  Encode form (name, value) and (name, filename, type) elements into
  multi-part/form-data. We don't actually need to know what we are
  uploading here, so just claim it's all text/plain.

Definition at line 32 of file alcaDQMUpload.py.

References filetype(), and str.

Referenced by BeautifulSoup.UnicodeDammit._detectEncoding(), trigger::TriggerEvent.addCollections(), uploads.uploader.get_upload_session_id(), util.getRunInfo.getFilesForRun(), marshall(), ConfigBuilder.ConfigBuilder.profileOptions(), util.getRunInfo.runGetDatasetsAvailable(), cmsswSequenceInfo.serve(), and uploadConditions.ConditionsUploader.uploadFile().

32 
33 def encode(args, files):
34  """
35  Encode form (name, value) and (name, filename, type) elements into
36  multi-part/form-data. We don't actually need to know what we are
37  uploading here, so just claim it's all text/plain.
38  """
39  boundary = '----------=_DQM_FILE_BOUNDARY_=-----------'
40  (body, crlf) = ('', '\r\n')
41  for (key, value) in args.items():
42  body += '--' + boundary + crlf
43  body += ('Content-disposition: form-data; name="%s"' % key) + crlf
44  body += crlf + str(value) + crlf
45  for (key, filename) in files.items():
46  body += '--' + boundary + crlf
47  body += ('Content-Disposition: form-data; name="%s"; filename="%s"'
48  % (key, os.path.basename(filename))) + crlf
49  body += ('Content-Type: %s' % filetype(filename)) + crlf
50  body += crlf + open(filename, "r").read() + crlf
51  body += '--' + boundary + '--' + crlf + crlf
52  return ('multipart/form-data; boundary=' + boundary, body)
str
#define str(s)
Definition: TestProcessor.cc:53
def alcaDQMUpload.filetype (   filename)

Definition at line 29 of file alcaDQMUpload.py.

Referenced by encode().

29 
30 def filetype(filename):
31  return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
def alcaDQMUpload.getURL (   filename,
  destination 
)

Definition at line 201 of file alcaDQMUpload.py.

References python.rootplot.root2matplotlib.replace().

Referenced by main().

201 
202 def getURL(filename, destination):
203  filename = filename.split("/")[-1]
204  regMC=re.compile('.*_R([0-9]*)__*')
205  if regMC.match(filename):
206  m = re.search('.*_R([0-9]*)(__.*).root',filename)
207  runNr = m.group(1)
208  dataset = m.group(2).replace("__","/")
209  else:
210  m = re.search('.*_R([0-9]*)(_?.*).root',filename)
211  runNr = m.group(1)
212  dataset = m.group(2).replace("__","/")
213  if dataset=="":
214  dataset="/Global/Online/ALL"
215  if not runNr:
216  runNr="1"
217  if (int(runNr)==1):
218  return destination+"start?workspace=summary;dataset="+dataset+";sampletype=offline_data"
219  else:
220  return destination+"start?workspace=summary;runnr="+runNr+";dataset="+dataset+";sampletype=online_data"
def alcaDQMUpload.main (   args)

Definition at line 238 of file alcaDQMUpload.py.

References checkFileName(), checkSSL(), getURL(), print(), registerFileAtLogServer(), and startUpload().

238 
239 def main(args):
240  global opts
241  parser = optparse.OptionParser(add_help_option=False)
242  parser.add_option("-h", "--help", action="callback", callback=print_help),
243  parser.add_option("", "--no-filename-check", dest="no_filename_check", default=False, action="store_true")
244  parser.add_option("-d", "--destination", dest="destination", default="", action="store")
245  parser.add_option("-t", "--tags", dest="tags", default="", action="store")
246  parser.add_option("-s", "--no-submission", dest="submission", default=True, action="store_false")
247  parser.add_option("-r", "--no-registration", dest="registration", default=True, action="store_false")
248  parser.add_option("","--ssl-key-file", dest="ssl_key_file", default="", action="store")
249  parser.add_option("","--ssl-cert-file", dest="ssl_cert_file", default="", action="store")
250  (opts, args) = parser.parse_args()
251  opts.abort = False
252 
253  if not opts.destination:
254  sys.stderr.write("no destination specified\n")
255  sys.exit(4)
256  checkSSL(opts)
257 
258  if len(args)==0:
259  sys.stderr.write("no input files specified\n")
260  sys.exit(1)
261  for fileName in args:
262  fileName=fileName.strip()
263  if not os.path.exists(fileName):
264  sys.stderr.write("file '%s' doesn't exist!\n" % fileName)
265  continue
266  if not opts.no_filename_check and not checkFileName(fileName):
267  continue
268  sys.stderr.write("file '%s' passed name check, upload will follow!\n" % fileName)
269  if opts.submission:
270  startUpload(opts.destination, fileName)
271  else:
272  sys.stdout.write("file '%s' would be uploaded to '%s'\n" % (fileName, opts.destination))
273  if opts.registration:
274  registerFileAtLogServer(fileName, opts.destination, opts.tags)
275  print("You should see the plots here: "+getURL(fileName, opts.destination))
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
alcaDQMUpload.registerFileAtLogServer
def registerFileAtLogServer
Definition: alcaDQMUpload.py:221
def alcaDQMUpload.marshall (   args,
  files,
  request 
) 
  Marshalls the arguments to the CGI script as multi-part/form-data,
  not the default application/x-www-form-url-encoded.  This improves
  the transfer of the large inputs and eases command line invocation
  of the CGI script.

Definition at line 53 of file alcaDQMUpload.py.

References encode(), and str.

Referenced by upload().

53 
54 def marshall(args, files, request):
55  """
56  Marshalls the arguments to the CGI script as multi-part/form-data,
57  not the default application/x-www-form-url-encoded. This improves
58  the transfer of the large inputs and eases command line invocation
59  of the CGI script.
60  """
61  (type, body) = encode(args, files)
62  request.add_header('Content-type', type)
63  request.add_header('Content-length', str(len(body)))
64  request.add_data(body)
str
#define str(s)
Definition: TestProcessor.cc:53
def alcaDQMUpload.print_help (   args)

Definition at line 92 of file alcaDQMUpload.py.

92 
93 def print_help(*args):
94  sys.stdout.write("\n")
95  sys.stdout.write("This scripts intends to do the upload of files to the DQM server. It\n")
96  sys.stdout.write("runs some basic checks on the file name as it is crucial to follow\n")
97  sys.stdout.write("the naming convention.\n")
98  sys.stdout.write("\n")
99  sys.stdout.write("Mandatory Option\n")
100  sys.stdout.write(" -d, --destination parameter to specify the DQM server\n")
101  sys.stdout.write("\n")
102  sys.stdout.write(" Proxy Options\n")
103  sys.stdout.write(" The script will try to find your grid proxy automatically. To\n")
104  sys.stdout.write(" do so, it checks $X509_* environment variables and your globus\n")
105  sys.stdout.write(" directory (~/.globus). To override this automatism, you can use\n")
106  sys.stdout.write(" the following two options:\n")
107  sys.stdout.write(" --ssl-key-file location of your private key\n")
108  sys.stdout.write(" --ssl-cert-file location of your public key\n")
109  sys.stdout.write("\n")
110  sys.stdout.write("Other Options\n")
111  sys.stdout.write(" -h, --help show this help message\n")
112  sys.stdout.write(" -s, --no-submission suppress the submission\n")
113  sys.stdout.write(" -r, --no-registration suppress the submission\n")
114  sys.stdout.write(" --no-filename-check omit the file name check\n")
115  sys.stdout.write("\n")
116  sys.exit(0)
def alcaDQMUpload.registerFileAtLogServer (   filename,
  destination,
  tags 
)

Definition at line 221 of file alcaDQMUpload.py.

References print(), and python.rootplot.root2matplotlib.replace().

Referenced by main().

221 
222 def registerFileAtLogServer(filename, destination, tags):
223  filename = filename.split("/")[-1]
224  regMC=re.compile('.*_R([0-9]*)__*')
225  if regMC.match(filename):
226  m = re.search('.*_R([0-9]*)(__.*).root',filename)
227  runNr = m.group(1)
228  dataset = m.group(2).replace("__","/")
229  else:
230  m = re.search('.*_R([0-9]*)(_?.*).root',filename)
231  runNr = m.group(1)
232  dataset = m.group(2).replace("__","/")
233  if dataset=="":
234  dataset="/Global/Online/ALL"
235  tempurl = "https://www-ekp.physik.uni-karlsruhe.de/~zeise/cgi-bin/register.py?run="+runNr+"&dataset="+dataset+"&filename="+filename+"&tags="+tags+"&instance="+destination
236  print("Link that is used to register: ", tempurl)
237  urllib.urlopen(tempurl)
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
alcaDQMUpload.registerFileAtLogServer
def registerFileAtLogServer
Definition: alcaDQMUpload.py:221
def alcaDQMUpload.startUpload (   url,
  filename 
)

Definition at line 179 of file alcaDQMUpload.py.

References mergeVDriftHistosByStation.file, print(), SiPixelLorentzAngle_cfi.read, and upload().

Referenced by main().

179 
180 def startUpload(url, filename):
181  global ssl_key_file
182  global ssl_cert_file
183 
184  print(url, filename)
185  try:
186  (headers, data) = \
187  upload(url,
188  { 'size': os.stat(filename).st_size,
189  'checksum': "md5:%s" % md5.new(file(filename).read()).hexdigest() },
190  { 'file': filename })
191  print('Status code: ', headers.get("Dqm-Status-Code", "None"))
192  print('Message: ', headers.get("Dqm-Status-Message", "None"))
193  print('Detail: ', headers.get("Dqm-Status-Detail", "None"))
194  print(data)
195  except urllib2.HTTPError as e:
196  print("ERROR", e)
197  print('Status code: ', e.hdrs.get("Dqm-Status-Code", "None"))
198  print('Message: ', e.hdrs.get("Dqm-Status-Message", "None"))
199  print('Detail: ', e.hdrs.get("Dqm-Status-Detail", "None"))
200  sys.exit(1)
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7
def alcaDQMUpload.upload (   url,
  args,
  files 
)

Definition at line 65 of file alcaDQMUpload.py.

References marshall().

Referenced by CommissioningHistosUsingDb.doUploadAnal(), CommissioningHistosUsingDb.doUploadConf(), startUpload(), ApvTimingHistosUsingDb.uploadConfigurations(), LatencyHistosUsingDb.uploadConfigurations(), and FineDelayHistosUsingDb.uploadConfigurations().

65 
66 def upload(url, args, files):
67  ident = "visDQMUpload DQMGUI/%s CMSSW/%s python/%s" % \
68  (os.getenv('DQMGUI_VERSION', '?'),
69  os.getenv('DQM_CMSSW_VERSION', os.getenv('CMSSW_VERSION', '?')),
70  "%d.%d.%d" % sys.version_info[:3])
71  cookie = None
72  if url.startswith("https:"):
73  authreq = urllib2.Request(url + '/authenticate')
74  authreq.add_header('User-agent', ident)
75  result = urllib2.build_opener(HTTPSCertAuthenticate()).open(authreq)
76  cookie = result.headers.get('Set-Cookie')
77  if not cookie:
78  raise RuntimeError("Did not receive authentication cookie")
79  cookie = cookie.split(";")[0]
80 
81  datareq = urllib2.Request(url + '/data/put')
82  datareq.add_header('Accept-encoding', 'gzip')
83  datareq.add_header('User-agent', ident)
84  if cookie:
85  datareq.add_header('Cookie', cookie)
86  marshall(args, files, datareq)
87  result = urllib2.build_opener().open(datareq)
88  data = result.read()
89  if result.headers.get ('Content-encoding', '') == 'gzip':
90  data = gzip.GzipFile (fileobj=StringIO(data)).read ()
91  return (result.headers, data)

Variable Documentation

alcaDQMUpload.HTTPS = httplib.HTTPS

Definition at line 14 of file alcaDQMUpload.py.

alcaDQMUpload.ssl_cert_file = None

Definition at line 19 of file alcaDQMUpload.py.

alcaDQMUpload.ssl_key_file = None

Definition at line 18 of file alcaDQMUpload.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5