CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Classes | Functions | Variables
das_client Namespace Reference

Classes

class  DASOptionParser
 
class  HTTPSClientAuthHandler
 

Functions

def check_auth
 
def check_glidein
 
def convert_time
 
def extract_value
 
def fullpath
 
def get_data
 
def get_value
 
def keys_attrs
 
def main
 
def prim_value
 
def print_from_cache
 
def print_summary
 
def size_format
 
def unique_filter
 
def x509
 

Variables

string __author__ = "Valentin Kuznetsov"
 
string DAS_CLIENT = 'das-client/1.1::python/%s.%s'
 
int EX__BASE = 64
 
int EX_CANTCREAT = 73
 
int EX_CONFIG = 78
 
int EX_DATAERR = 65
 
int EX_IOERR = 74
 
int EX_NOHOST = 68
 
int EX_NOINPUT = 66
 
int EX_NOPERM = 77
 
int EX_NOUSER = 67
 
int EX_OK = 0
 
int EX_OSERR = 71
 
int EX_OSFILE = 72
 
int EX_PROTOCOL = 76
 
int EX_SOFTWARE = 70
 
int EX_TEMPFAIL = 75
 
int EX_UNAVAILABLE = 69
 
int EX_USAGE = 64
 

Function Documentation

def das_client.check_auth (   key)

Definition at line 102 of file das_client.py.

References print().

Referenced by main().

102 
103 def check_auth(key):
104  "Check if user runs das_client with key/cert and warn users to switch"
105  if not key:
106  msg = "WARNING: das_client is running without user credentials/X509 proxy, create proxy via 'voms-proxy-init -voms cms -rfc'"
107  print(msg, file=sys.stderr)
das_client.check_auth
def check_auth
Definition: das_client.py:102
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def das_client.check_glidein ( )

Definition at line 94 of file das_client.py.

References print().

Referenced by main().

94 
95 def check_glidein():
96  "Check glideine environment and exit if it is set"
97  glidein = os.environ.get('GLIDEIN_CMSSite', '')
98  if glidein:
99  msg = "ERROR: das_client is running from GLIDEIN environment, it is prohibited"
100  print(msg)
101  sys.exit(EX__BASE)
das_client.check_glidein
def check_glidein
Definition: das_client.py:94
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def das_client.convert_time (   val)

Definition at line 176 of file das_client.py.

Referenced by extract_value().

176 
177 def convert_time(val):
178  "Convert given timestamp into human readable format"
179  if isinstance(val, int) or isinstance(val, float):
180  return time.strftime('%d/%b/%Y_%H:%M:%S_GMT', time.gmtime(val))
181  return val
das_client.convert_time
def convert_time
Definition: das_client.py:176
def das_client.extract_value (   row,
  key,
  base = 10 
) 
Generator which extracts row[key] value

Definition at line 233 of file das_client.py.

References convert_time(), and size_format().

Referenced by get_value().

233 
234 def extract_value(row, key, base=10):
235  """Generator which extracts row[key] value"""
236  if isinstance(row, dict) and key in row:
237  if key == 'creation_time':
238  row = convert_time(row[key])
239  elif key == 'size':
240  row = size_format(row[key], base)
241  else:
242  row = row[key]
243  yield row
244  if isinstance(row, list) or isinstance(row, GeneratorType):
245  for item in row:
246  for vvv in extract_value(item, key, base):
247  yield vvv
das_client.convert_time
def convert_time
Definition: das_client.py:176
das_client.size_format
def size_format
Definition: das_client.py:182
das_client.extract_value
def extract_value
Definition: das_client.py:233
def das_client.fullpath (   path)

Definition at line 267 of file das_client.py.

References get_data().

Referenced by dqm::implementation::IBooker.bookME(), edm::storage::LStoreStorageMaker.check(), edm::storage::XrdStorageMaker.check(), GEMEfficiencyHarvester.doEfficiency(), GEMEfficiencyHarvester.doResolution(), SiStripQualityChecker.fillDetectorStatusAtLumi(), edm::storage::LocalFileSystem.findCachePath(), edm::storage::LocalFileSystem.findMount(), DQMService.flushStandalone(), get_data(), CommonAnalyzer.getList(), evf::EvFDaqDirector.getLumisectionToStart(), LegacyIOHelper.getMEName(), evf::EvFDaqDirector.getNFilesFromEoLS(), CommonAnalyzer.getObject(), edm::storage::LocalFileSystem.isLocalPath(), keys_attrs(), MuonErrorMatrix.MuonErrorMatrix(), edm::storage::LStoreStorageMaker.open(), edm::storage::XrdStorageMaker.open(), and edm::storage::XrdStorageMaker.stagein().

267 
268 def fullpath(path):
269  "Expand path to full path"
270  if path and path[0] == '~':
271  path = path.replace('~', '')
272  path = path[1:] if path[0] == '/' else path
273  path = os.path.join(os.environ['HOME'], path)
274  return path
das_client.fullpath
def fullpath
Definition: das_client.py:267
def das_client.get_data (   host,
  query,
  idx,
  limit,
  debug,
  threshold = 300,
  ckey = None,
  cert = None,
  capath = None,
  qcache = 0,
  das_headers = True 
) 
Contact DAS server and retrieve data for given DAS query

Definition at line 276 of file das_client.py.

References fullpath(), print(), and str.

Referenced by dataset.Dataset.__getData(), cmsswVersionTools.PickRelValInputFiles.apply(), electronDataDiscovery.common_search(), dasFileQuery.dasFileQuery(), dataset.dasquery(), util.getRunInfo.dasQuery(), fullpath(), edmPickEvents.getFileNames_das_client(), and main().

276 
277  cert=None, capath=None, qcache=0, das_headers=True):
278  """Contact DAS server and retrieve data for given DAS query"""
279  params = {'input':query, 'idx':idx, 'limit':limit}
280  if qcache:
281  params['qcache'] = qcache
282  path = '/das/cache'
283  pat = re.compile('http[s]{0,1}://')
284  if not pat.match(host):
285  msg = 'Invalid hostname: %s' % host
286  raise Exception(msg)
287  url = host + path
288  client = '%s (%s)' % (DAS_CLIENT, os.environ.get('USER', ''))
289  headers = {"Accept": "application/json", "User-Agent": client}
290  encoded_data = urllib.urlencode(params, doseq=True)
291  url += '?%s' % encoded_data
292  req = urllib2.Request(url=url, headers=headers)
293  if ckey and cert:
294  ckey = fullpath(ckey)
295  cert = fullpath(cert)
296  http_hdlr = HTTPSClientAuthHandler(ckey, cert, capath, debug)
297  elif cert and capath:
298  cert = fullpath(cert)
299  http_hdlr = HTTPSClientAuthHandler(ckey, cert, capath, debug)
300  else:
301  http_hdlr = urllib2.HTTPHandler(debuglevel=debug)
302  proxy_handler = urllib2.ProxyHandler({})
303  cookie_jar = cookielib.CookieJar()
304  cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)
305  try:
306  opener = urllib2.build_opener(http_hdlr, proxy_handler, cookie_handler)
307  fdesc = opener.open(req)
308  data = fdesc.read()
309  fdesc.close()
310  except urllib2.HTTPError as error:
311  print(error.read())
312  sys.exit(1)
313 
314  pat = re.compile(r'^[a-z0-9]{32}')
315  if data and isinstance(data, str) and pat.match(data) and len(data) == 32:
316  pid = data
317  else:
318  pid = None
319  iwtime = 2 # initial waiting time in seconds
320  wtime = 20 # final waiting time in seconds
321  sleep = iwtime
322  time0 = time.time()
323  while pid:
324  params.update({'pid':data})
325  encoded_data = urllib.urlencode(params, doseq=True)
326  url = host + path + '?%s' % encoded_data
327  req = urllib2.Request(url=url, headers=headers)
328  try:
329  fdesc = opener.open(req)
330  data = fdesc.read()
331  fdesc.close()
332  except urllib2.HTTPError as err:
333  return {"status":"fail", "reason":str(err)}
334  if data and isinstance(data, str) and pat.match(data) and len(data) == 32:
335  pid = data
336  else:
337  pid = None
338  time.sleep(sleep)
339  if sleep < wtime:
340  sleep *= 2
341  elif sleep == wtime:
342  sleep = iwtime # start new cycle
343  else:
344  sleep = wtime
345  if (time.time()-time0) > threshold:
346  reason = "client timeout after %s sec" % int(time.time()-time0)
347  return {"status":"fail", "reason":reason}
348  jsondict = json.loads(data)
349  return jsondict
das_client.fullpath
def fullpath
Definition: das_client.py:267
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
das_client.HTTPSClientAuthHandler
Definition: das_client.py:52
str
#define str(s)
Definition: TestProcessor.cc:53
def das_client.get_value (   data,
  filters,
  base = 10 
) 
Filter data from a row for given list of filters

Definition at line 248 of file das_client.py.

References extract_value().

Referenced by cmsswVersionTools.PickRelValInputFiles.apply(), edmPickEvents.getFileNames_das_client(), and main().

248 
249 def get_value(data, filters, base=10):
250  """Filter data from a row for given list of filters"""
251  for ftr in filters:
252  if ftr.find('>') != -1 or ftr.find('<') != -1 or ftr.find('=') != -1:
253  continue
254  row = dict(data)
255  values = []
256  keys = ftr.split('.')
257  for key in keys:
258  val = [v for v in extract_value(row, key, base)]
259  if key == keys[-1]: # we collect all values at last key
260  values += [json.dumps(i) for i in val]
261  else:
262  row = val
263  if len(values) == 1:
264  yield values[0]
265  else:
266  yield values
das_client.extract_value
def extract_value
Definition: das_client.py:233
das_client.get_value
def get_value
Definition: das_client.py:248
def das_client.keys_attrs (   lkey,
  oformat,
  host,
  ckey,
  cert,
  debug = 0 
)

Definition at line 387 of file das_client.py.

References fullpath(), and print().

Referenced by main().

387 
388 def keys_attrs(lkey, oformat, host, ckey, cert, debug=0):
389  "Contact host for list of key/attributes pairs"
390  url = '%s/das/keys?view=json' % host
391  headers = {"Accept": "application/json", "User-Agent": DAS_CLIENT}
392  req = urllib2.Request(url=url, headers=headers)
393  if ckey and cert:
394  ckey = fullpath(ckey)
395  cert = fullpath(cert)
396  http_hdlr = HTTPSClientAuthHandler(ckey, cert, debug)
397  else:
398  http_hdlr = urllib2.HTTPHandler(debuglevel=debug)
399  proxy_handler = urllib2.ProxyHandler({})
400  cookie_jar = cookielib.CookieJar()
401  cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)
402  opener = urllib2.build_opener(http_hdlr, proxy_handler, cookie_handler)
403  fdesc = opener.open(req)
404  data = json.load(fdesc)
405  fdesc.close()
406  if oformat.lower() == 'json':
407  if lkey == 'all':
408  print(json.dumps(data))
409  else:
410  print(json.dumps({lkey:data[lkey]}))
411  return
412  for key, vdict in data.items():
413  if lkey == 'all':
414  pass
415  elif lkey != key:
416  continue
417  print()
418  print("DAS key:", key)
419  for attr, examples in vdict.items():
420  prefix = ' '
421  print('%s%s' % (prefix, attr))
422  for item in examples:
423  print('%s%s%s' % (prefix, prefix, item))
das_client.fullpath
def fullpath
Definition: das_client.py:267
das_client.keys_attrs
def keys_attrs
Definition: das_client.py:387
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
das_client.HTTPSClientAuthHandler
Definition: das_client.py:52
def das_client.main ( ) 
Main function

Definition at line 424 of file das_client.py.

References check_auth(), check_glidein(), spr.find(), get_data(), get_value(), join(), keys_attrs(), log, prim_value(), print(), print_from_cache(), print_summary(), size_format(), str, and unique_filter().

424 
425 def main():
426  """Main function"""
427  optmgr = DASOptionParser()
428  opts, _ = optmgr.get_opt()
429  host = opts.host
430  debug = opts.verbose
431  query = opts.query
432  idx = opts.idx
433  limit = opts.limit
434  thr = opts.threshold
435  ckey = opts.ckey
436  cert = opts.cert
437  capath = opts.capath
438  base = opts.base
439  qcache = opts.qcache
440  check_glidein()
441  check_auth(ckey)
442  if opts.keys_attrs:
443  keys_attrs(opts.keys_attrs, opts.format, host, ckey, cert, debug)
444  return
445  if not query:
446  print('Input query is missing')
447  sys.exit(EX_USAGE)
448  if opts.format == 'plain':
449  jsondict = get_data(host, query, idx, limit, debug, thr, ckey, cert, capath, qcache)
450  cli_msg = jsondict.get('client_message', None)
451  if cli_msg:
452  print("DAS CLIENT WARNING: %s" % cli_msg)
453  if 'status' not in jsondict and opts.cache:
454  print_from_cache(opts.cache, query)
455  if 'status' not in jsondict:
456  print('DAS record without status field:\n%s' % jsondict)
457  sys.exit(EX_PROTOCOL)
458  if jsondict["status"] != 'ok' and opts.cache:
459  print_from_cache(opts.cache, query)
460  if jsondict['status'] != 'ok':
461  print("status: %s, reason: %s" \
462  % (jsondict.get('status'), jsondict.get('reason', 'N/A')))
463  if opts.retry:
464  found = False
465  for attempt in xrange(1, int(opts.retry)):
466  interval = log(attempt)**5
467  print("Retry in %5.3f sec" % interval)
468  time.sleep(interval)
469  data = get_data(host, query, idx, limit, debug, thr, ckey, cert, capath, qcache)
470  jsondict = json.loads(data)
471  if jsondict.get('status', 'fail') == 'ok':
472  found = True
473  break
474  else:
475  sys.exit(EX_TEMPFAIL)
476  if not found:
477  sys.exit(EX_TEMPFAIL)
478  nres = jsondict.get('nresults', 0)
479  if not limit:
480  drange = '%s' % nres
481  else:
482  drange = '%s-%s out of %s' % (idx+1, idx+limit, nres)
483  if opts.limit:
484  msg = "\nShowing %s results" % drange
485  msg += ", for more results use --idx/--limit options\n"
486  print(msg)
487  mongo_query = jsondict.get('mongo_query', {})
488  unique = False
489  fdict = mongo_query.get('filters', {})
490  filters = fdict.get('grep', [])
491  aggregators = mongo_query.get('aggregators', [])
492  if 'unique' in fdict.keys():
493  unique = True
494  if filters and not aggregators:
495  data = jsondict['data']
496  if isinstance(data, dict):
497  rows = [r for r in get_value(data, filters, base)]
498  print(' '.join(rows))
499  elif isinstance(data, list):
500  if unique:
501  data = unique_filter(data)
502  for row in data:
503  rows = [r for r in get_value(row, filters, base)]
504  types = [type(r) for r in rows]
505  if len(types)>1: # mixed types print as is
506  print(' '.join([str(r) for r in rows]))
507  elif isinstance(rows[0], list):
508  out = set()
509  for item in rows:
510  for elem in item:
511  out.add(elem)
512  print(' '.join(out))
513  else:
514  print(' '.join(rows))
515  else:
516  print(json.dumps(jsondict))
517  elif aggregators:
518  data = jsondict['data']
519  if unique:
520  data = unique_filter(data)
521  for row in data:
522  if row['key'].find('size') != -1 and \
523  row['function'] == 'sum':
524  val = size_format(row['result']['value'], base)
525  else:
526  val = row['result']['value']
527  print('%s(%s)=%s' \
528  % (row['function'], row['key'], val))
529  else:
530  data = jsondict['data']
531  if isinstance(data, list):
532  old = None
533  val = None
534  for row in data:
535  prim_key = row.get('das', {}).get('primary_key', None)
536  if prim_key == 'summary':
537  print_summary(row)
538  return
539  val = prim_value(row)
540  if not opts.limit:
541  if val != old:
542  print(val)
543  old = val
544  else:
545  print(val)
546  if val != old and not opts.limit:
547  print(val)
548  elif isinstance(data, dict):
549  print(prim_value(data))
550  else:
551  print(data)
552  else:
553  jsondict = get_data(\
554  host, query, idx, limit, debug, thr, ckey, cert, capath, qcache)
555  print(json.dumps(jsondict))
556 
557 #
558 # main
#
das_client.main
def main
Definition: das_client.py:424
das_client.unique_filter
def unique_filter
Definition: das_client.py:203
log
static std::vector< std::string > checklist log
Definition: LHERunInfoProduct.cc:179
das_client.size_format
def size_format
Definition: das_client.py:182
das_client.check_auth
def check_auth
Definition: das_client.py:102
das_client.check_glidein
def check_glidein
Definition: das_client.py:94
spr::find
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:19
das_client.get_data
def get_data
Definition: das_client.py:276
das_client.keys_attrs
def keys_attrs
Definition: das_client.py:387
das_client.print_summary
def print_summary
Definition: das_client.py:365
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
das_client.DASOptionParser
Definition: das_client.py:108
das_client.get_value
def get_value
Definition: das_client.py:248
das_client.print_from_cache
def print_from_cache
Definition: das_client.py:378
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
das_client.prim_value
def prim_value
Definition: das_client.py:350
str
#define str(s)
Definition: TestProcessor.cc:53
def das_client.prim_value (   row) 
Extract primary key value from DAS record

Definition at line 350 of file das_client.py.

Referenced by main().

350 
351 def prim_value(row):
352  """Extract primary key value from DAS record"""
353  prim_key = row['das']['primary_key']
354  if prim_key == 'summary':
355  return row.get(prim_key, None)
356  key, att = prim_key.split('.')
357  if isinstance(row[key], list):
358  for item in row[key]:
359  if att in item:
360  return item[att]
361  else:
362  if key in row:
363  if att in row[key]:
364  return row[key][att]
das_client.prim_value
def prim_value
Definition: das_client.py:350
def das_client.print_from_cache (   cache,
  query 
)

Definition at line 378 of file das_client.py.

References beamvalidation.exit(), join(), print(), and SiPixelLorentzAngle_cfi.read.

Referenced by main().

378 
379 def print_from_cache(cache, query):
380  "print the list of files reading it from cache"
381  data = open(cache).read()
382  jsondict = json.loads(data)
383  if query in jsondict:
384  print("\n".join(jsondict[query]))
385  exit(0)
386  exit(1)
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7
das_client.print_from_cache
def print_from_cache
Definition: das_client.py:378
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def das_client.print_summary (   rec)

Definition at line 365 of file das_client.py.

References SiStripPI.max, and print().

Referenced by main().

365 
366 def print_summary(rec):
367  "Print summary record information on stdout"
368  if 'summary' not in rec:
369  msg = 'Summary information is not found in record:\n', rec
370  raise Exception(msg)
371  for row in rec['summary']:
372  keys = [k for k in row.keys()]
373  maxlen = max([len(k) for k in keys])
374  for key, val in row.items():
375  pkey = '%s%s' % (key, ' '*(maxlen-len(key)))
376  print('%s: %s' % (pkey, val))
377  print()
das_client.print_summary
def print_summary
Definition: das_client.py:365
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def das_client.size_format (   uinput,
  ibase = 0 
) 
Format file size utility, it converts file size into KB, MB, GB, TB, PB units

Definition at line 182 of file das_client.py.

Referenced by extract_value(), and main().

182 
183 def size_format(uinput, ibase=0):
184  """
185  Format file size utility, it converts file size into KB, MB, GB, TB, PB units
186  """
187  if not ibase:
188  return uinput
189  try:
190  num = float(uinput)
191  except Exception as _exc:
192  return uinput
193  if ibase == 2.: # power of 2
194  base = 1024.
195  xlist = ['', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']
196  else: # default base is 10
197  base = 1000.
198  xlist = ['', 'KB', 'MB', 'GB', 'TB', 'PB']
199  for xxx in xlist:
200  if num < base:
201  return "%3.1f%s" % (num, xxx)
202  num /= base
das_client.size_format
def size_format
Definition: das_client.py:182
def das_client.unique_filter (   rows) 
Unique filter drop duplicate rows.

Definition at line 203 of file das_client.py.

Referenced by main().

203 
204 def unique_filter(rows):
205  """
206  Unique filter drop duplicate rows.
207  """
208  old_row = {}
209  row = None
210  for row in rows:
211  row_data = dict(row)
212  try:
213  del row_data['_id']
214  del row_data['das']
215  del row_data['das_id']
216  del row_data['cache_id']
217  except:
218  pass
219  old_data = dict(old_row)
220  try:
221  del old_data['_id']
222  del old_data['das']
223  del old_data['das_id']
224  del old_data['cache_id']
225  except:
226  pass
227  if row_data == old_data:
228  continue
229  if old_row:
230  yield old_row
231  old_row = row
232  yield row
das_client.unique_filter
def unique_filter
Definition: das_client.py:203
def das_client.x509 ( )

Definition at line 85 of file das_client.py.

85 
86 def x509():
87  "Helper function to get x509 either from env or tmp file"
88  proxy = os.environ.get('X509_USER_PROXY', '')
89  if not proxy:
90  proxy = '/tmp/x509up_u%s' % pwd.getpwuid( os.getuid() ).pw_uid
91  if not os.path.isfile(proxy):
92  return ''
93  return proxy
das_client.x509
def x509
Definition: das_client.py:85

Variable Documentation

string das_client.__author__ = "Valentin Kuznetsov"

Definition at line 9 of file das_client.py.

string das_client.DAS_CLIENT = 'das-client/1.1::python/%s.%s'

Definition at line 18 of file das_client.py.

int das_client.EX__BASE = 64

Definition at line 35 of file das_client.py.

int das_client.EX_CANTCREAT = 73

Definition at line 45 of file das_client.py.

int das_client.EX_CONFIG = 78

Definition at line 50 of file das_client.py.

int das_client.EX_DATAERR = 65

Definition at line 37 of file das_client.py.

int das_client.EX_IOERR = 74

Definition at line 46 of file das_client.py.

int das_client.EX_NOHOST = 68

Definition at line 40 of file das_client.py.

int das_client.EX_NOINPUT = 66

Definition at line 38 of file das_client.py.

int das_client.EX_NOPERM = 77

Definition at line 49 of file das_client.py.

int das_client.EX_NOUSER = 67

Definition at line 39 of file das_client.py.

int das_client.EX_OK = 0

Definition at line 34 of file das_client.py.

int das_client.EX_OSERR = 71

Definition at line 43 of file das_client.py.

int das_client.EX_OSFILE = 72

Definition at line 44 of file das_client.py.

int das_client.EX_PROTOCOL = 76

Definition at line 48 of file das_client.py.

int das_client.EX_SOFTWARE = 70

Definition at line 42 of file das_client.py.

int das_client.EX_TEMPFAIL = 75

Definition at line 47 of file das_client.py.

int das_client.EX_UNAVAILABLE = 69

Definition at line 41 of file das_client.py.

int das_client.EX_USAGE = 64

Definition at line 36 of file das_client.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5