test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
edmPickEvents Namespace Reference

Classes

class  Event
 Event helper class ##. More...
 

Functions

def convert_time
 
def extract_value
 
def fullCPMpath
 
def get_value
 
def getFileNames
 Subroutines ##. More...
 
def guessEmail
 
def setupCrabDict
 
def size_format
 

Variables

tuple colonRE = re.compile(r':')
 
string command = 'edmCopyPickMerge outputFile=%s.root \\\n eventsToProcess=%s \\\n inputFiles=%s'
 
tuple commentRE = re.compile(r'#.+$')
 
tuple crabDict = setupCrabDict(options)
 
string crabTemplate
 
string default = 'pickevents'
 
tuple email = guessEmail()
 

Main Program

More...
 
tuple event = Event(piece)
 
tuple eventFiles = getFileNames(event)
 
list eventList = []
 
list eventPurgeList = []
 
string eventsToProcess = '\n'
 
list files = []
 Interactive ##. More...
 
tuple fileSet = set()
 
string help
 
tuple json = LumiList(lumis = runsAndLumis)
 
tuple line = commentRE.sub('', line)
 
tuple parser
 
list runsAndLumis = [ (event.run, event.lumi) for event in eventList]
 CRAB ##. More...
 
tuple source = open(args[0], 'r')
 
tuple target = open(crabDict['runEvent'], 'w')
 
list uniqueFiles = []
 

Function Documentation

def edmPickEvents.convert_time (   val)

Definition at line 52 of file edmPickEvents.py.

Referenced by extract_value().

52 
53 def convert_time(val):
54  "Convert given timestamp into human readable format"
55  if isinstance(val, int) or isinstance(val, float):
56  return time.strftime('%d/%b/%Y_%H:%M:%S_GMT', time.gmtime(val))
57  return val
def edmPickEvents.extract_value (   row,
  key,
  base = 10 
) 
Generator which extracts row[key] value

Definition at line 79 of file edmPickEvents.py.

References convert_time(), and size_format().

Referenced by get_value().

79 
80 def extract_value(row, key, base=10):
81  """Generator which extracts row[key] value"""
82  if isinstance(row, dict) and key in row:
83  if key == 'creation_time':
84  row = convert_time(row[key])
85  elif key == 'size':
86  row = size_format(row[key], base)
87  else:
88  row = row[key]
89  yield row
90  if isinstance(row, list) or isinstance(row, GeneratorType):
91  for item in row:
92  for vvv in extract_value(item, key, base):
93  yield vvv
94 
def edmPickEvents.fullCPMpath ( )

Definition at line 176 of file edmPickEvents.py.

Referenced by setupCrabDict().

176 
177 def fullCPMpath():
178  base = os.environ.get ('CMSSW_BASE')
179  if not base:
180  raise RuntimeError("CMSSW Environment not set")
181  retval = "%s/src/PhysicsTools/Utilities/configuration/copyPickMerge_cfg.py" \
182  % base
183  if os.path.exists (retval):
184  return retval
185  base = os.environ.get ('CMSSW_RELEASE_BASE')
186  retval = "%s/src/PhysicsTools/Utilities/configuration/copyPickMerge_cfg.py" \
187  % base
188  if os.path.exists (retval):
189  return retval
190  raise RuntimeError("Could not find copyPickMerge_cfg.py")
def edmPickEvents.get_value (   data,
  filters,
  base = 10 
) 
Filter data from a row for given list of filters

Definition at line 95 of file edmPickEvents.py.

References cmsPerfStripChart.dict, and extract_value().

Referenced by getFileNames().

95 
96 def get_value(data, filters, base=10):
97  """Filter data from a row for given list of filters"""
98  for ftr in filters:
99  if ftr.find('>') != -1 or ftr.find('<') != -1 or ftr.find('=') != -1:
100  continue
101  row = dict(data)
102  values = []
103  keys = ftr.split('.')
104  for key in keys:
105  val = [v for v in extract_value(row, key, base)]
106  if key == keys[-1]: # we collect all values at last key
107  values += [json.dumps(i) for i in val]
108  else:
109  row = val
110  if len(values) == 1:
111  yield values[0]
112  else:
113  yield values
114 
115 
def edmPickEvents.getFileNames (   event)

Subroutines ##.

Definition at line 149 of file edmPickEvents.py.

References get_value().

149 
150 def getFileNames (event):
151  files = []
152 
153  query = "file dataset=%(dataset)s run=%(run)i lumi=%(lumi)i | grep file.name" % event
154  p = Popen('das_client --format json --query "%s"'%(query), stdout=PIPE,shell=True)
155  pipe=p.stdout.read()
156  tupleP = os.waitpid(p.pid, 0)
157 
158  jsondict = json.loads(pipe)#das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, False)
159  status = jsondict['status']
160  if status != 'ok':
161  print "DAS query status: %s"%(status)
162  return files
163 
164  mongo_query = jsondict['mongo_query']
165  filters = mongo_query['filters']
166  data = jsondict['data']
167 
168  files = []
169  for row in data:
170  file = [r for r in get_value(row, filters['grep'])][0]
171  if len(file) > 0 and not file in files:
172  files.append(file)
173 
174  return files
175 
edmPickEvents.getFileNames
def getFileNames
Subroutines ##.
Definition: edmPickEvents.py:149
def edmPickEvents.guessEmail ( )

Definition at line 191 of file edmPickEvents.py.

References join(), and split.

191 
192 def guessEmail():
193  return '%s@%s' % (commands.getoutput ('whoami'),
194  '.'.join(commands.getoutput('hostname').split('.')[-2:]))
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
split
double split
Definition: MVATrainer.cc:139
def edmPickEvents.setupCrabDict (   options)

Definition at line 195 of file edmPickEvents.py.

References fullCPMpath().

195 
196 def setupCrabDict (options):
197  date = datetime.now().strftime('%Y%m%d_%H%M%S')
198  crab = {}
199  base = options.base
200  crab['runEvent'] = '%s_runEvents.txt' % base
201  crab['copyPickMerge'] = fullCPMpath()
202  crab['output'] = '%s.root' % base
203  crab['crabcfg'] = '%s_crab.py' % base
204  crab['json'] = '%s.json' % base
205  crab['dataset'] = Event.dataset
206  crab['email'] = options.email
207  crab['WorkArea'] = date
208  if options.crabCondor:
209  crab['scheduler'] = 'condor'
210 # crab['useServer'] = ''
211  else:
212  crab['scheduler'] = 'remoteGlidein'
213 # crab['useServer'] = 'use_server = 1'
214  crab['useServer'] = ''
215  return crab
216 
# crab template
def edmPickEvents.size_format (   uinput,
  ibase = 0 
) 
Format file size utility, it converts file size into KB, MB, GB, TB, PB units

Definition at line 58 of file edmPickEvents.py.

Referenced by extract_value().

58 
59 def size_format(uinput, ibase=0):
60  """
61  Format file size utility, it converts file size into KB, MB, GB, TB, PB units
62  """
63  if not ibase:
64  return uinput
65  try:
66  num = float(uinput)
67  except Exception as _exc:
68  return uinput
69  if ibase == 2.: # power of 2
70  base = 1024.
71  xlist = ['', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']
72  else: # default base is 10
73  base = 1000.
74  xlist = ['', 'KB', 'MB', 'GB', 'TB', 'PB']
75  for xxx in xlist:
76  if num < base:
77  return "%3.1f%s" % (num, xxx)
78  num /= base

Variable Documentation

tuple edmPickEvents.colonRE = re.compile(r':')

Definition at line 301 of file edmPickEvents.py.

string edmPickEvents.command = 'edmCopyPickMerge outputFile=%s.root \\\n eventsToProcess=%s \\\n inputFiles=%s'

Definition at line 385 of file edmPickEvents.py.

tuple edmPickEvents.commentRE = re.compile(r'#.+$')

Definition at line 300 of file edmPickEvents.py.

tuple edmPickEvents.crabDict = setupCrabDict(options)

Definition at line 342 of file edmPickEvents.py.

string edmPickEvents.crabTemplate

Definition at line 217 of file edmPickEvents.py.

string edmPickEvents.default = 'pickevents'

Definition at line 273 of file edmPickEvents.py.

tuple edmPickEvents.email = guessEmail()

Main Program

Definition at line 268 of file edmPickEvents.py.

tuple edmPickEvents.event = Event(piece)

Definition at line 307 of file edmPickEvents.py.

tuple edmPickEvents.eventFiles = getFileNames(event)

Definition at line 365 of file edmPickEvents.py.

list edmPickEvents.eventList = []

Definition at line 302 of file edmPickEvents.py.

list edmPickEvents.eventPurgeList = []

Definition at line 363 of file edmPickEvents.py.

string edmPickEvents.eventsToProcess = '\n'

Definition at line 340 of file edmPickEvents.py.

list edmPickEvents.files = []

Interactive ##.

Definition at line 362 of file edmPickEvents.py.

tuple edmPickEvents.fileSet = set()

Definition at line 375 of file edmPickEvents.py.

Referenced by TauDQMFileLoader.endRun(), and TauDQMFileLoader.TauDQMFileLoader().

string edmPickEvents.help
Initial value:
1 = """
2 How to use:
3 
4 edmPickEvent.py dataset run1:lumi1:event1 run2:lumi2:event2
5 
6 - or -
7 
8 edmPickEvent.py dataset listOfEvents.txt
9 
10 listOfEvents is a text file:
11 # this line is ignored as a comment
12 # since '#' is a valid comment character
13 run1 lumi_section1 event1
14 run2 lumi_section2 event2
15 
16 For example:
17 # run lum event
18 46968 2 4
19 47011 105 23
20 47011 140 12312
21 
22 run, lumi_section, and event are integers that you can get from
23 edm::Event(Auxiliary)
24 
25 dataset: it just a name of the physics dataset, if you don't know exact name
26  you can provide a mask, e.g.: *QCD*RAW
27 
28 For updated information see Wiki:
29 https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookPickEvents
30 """

Definition at line 20 of file edmPickEvents.py.

tuple edmPickEvents.json = LumiList(lumis = runsAndLumis)

Definition at line 339 of file edmPickEvents.py.

tuple edmPickEvents.line = commentRE.sub('', line)

Definition at line 315 of file edmPickEvents.py.

tuple edmPickEvents.parser
Initial value:
1 = optparse.OptionParser("Usage: %prog [options] dataset events_or_events.txt", description='''This program
2 facilitates picking specific events from a data set. For full details, please visit
3 https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookPickEvents ''')

Definition at line 269 of file edmPickEvents.py.

list edmPickEvents.runsAndLumis = [ (event.run, event.lumi) for event in eventList]

CRAB ##.

Definition at line 338 of file edmPickEvents.py.

string edmPickEvents.source = open(args[0], 'r')

Definition at line 313 of file edmPickEvents.py.

tuple edmPickEvents.target = open(crabDict['runEvent'], 'w')

Definition at line 344 of file edmPickEvents.py.

list edmPickEvents.uniqueFiles = []

Definition at line 376 of file edmPickEvents.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5