CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Classes | Functions | Variables
utils_v2 Namespace Reference

Classes

class  Chi2Test
 
class  ComparisonError
 
class  KolmogorovTest
 
class  StatisticalTest
 StatisticalTests. More...
 

Functions

def auth_download_file
 
def auth_wget
 -----------------— Recursife file downloader --------------------— More...
 
def check_disk_for_space
 
def get_relval_cmssw_version
 
def get_relval_id
 
def get_relval_max_version
 
def get_relval_version
 -------------—— Make files pairs: RelVal utils ---------------—— More...
 
def get_relvaldata_cmssw_version
 
def get_relvaldata_id
 -----------—— Make files pairs: RelValData utils --------------—— More...
 
def get_relvaldata_max_version
 
def get_relvaldata_version
 
def get_size_to_download
 
def get_version
 
def init_database
 Utils. More...
 
def is_relvaldata
 --------------------— Make file pairs -----------------------— More...
 
def make_file_pairs
 
def recursive_search_online
 
def search_on_disk
 
def show_status_bar
 

Variables

dictionary comparison_errors
 Exception definitions. More...
 
dictionary tests = {KolmogorovTest.name: KolmogorovTest, Chi2Test.name: Chi2Test}
 

Function Documentation

def utils_v2.auth_download_file (   url,
  chunk_size = 1048576 
)

Definition at line 200 of file utils_v2.py.

References join(), and print().

201 def auth_download_file(url, chunk_size=1048576):
202  filename = basename(url)
203  file_path = join(auth_download_file.work_dir, filename)
204 
205  file = open(file_path, 'wb')
206  opener = build_opener(X509CertOpen())
207  url_file = opener.open(Request(url))
208  chunk = url_file.read(chunk_size)
209  while chunk:
210  file.write(chunk)
211  auth_download_file.q.put((1,)) # reports, that downloaded 1MB
212  chunk = url_file.read(chunk_size)
213  print('\rDownloaded: %s ' % (filename,))
214  file.close()
215 
def auth_download_file
Definition: utils_v2.py:200
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def utils_v2.auth_wget (   url)

-----------------— Recursife file downloader --------------------—

Definition at line 186 of file utils_v2.py.

References beamvalidation.exit(), print(), and SiPixelLorentzAngle_cfi.read.

Referenced by recursive_search_online().

187 def auth_wget(url):
188  try:
189  opener = build_opener(X509CertOpen())
190  return opener.open(Request(url)).read()
191  except HTTPError as e:
192  print('\nError: DQM GUI is temporarily unavailable. Probably maintainance hours. '+\
193  'Please try again later. Original error message: ``%s``. \nExiting...\n' % (e,))
194  exit()
195  except BadStatusLine as e:
196  print('\nYou do not have permissions to access DQM GUI. Please check if your certificates '+\
197  'in ``~/.globus`` directory are configured correctly. Exitting...')
198  exit()
199 
def auth_wget
-----------------— Recursife file downloader --------------------—
Definition: utils_v2.py:186
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def utils_v2.check_disk_for_space (   work_path,
  size_needed 
)
Checks afs file system for space.

Definition at line 462 of file utils_v2.py.

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

463 def check_disk_for_space(work_path, size_needed):
464  '''Checks afs file system for space.'''
465  pass
466  # try:
467  # fs_proc = subprocess.Popen(['fs', 'listquota', work_path], stdout=subprocess.PIPE)
468  # except OSError:
469  # return
470  # fs_response = fs_proc.communicate()[0]
471  # quota, used = re.findall('([\d]+)', fs_response)[:2]
472  # free_space = int(quota) - int(used)
473  # if free_space * 1024 < size_needed:
474  # print '\nNot enougth free space on disk.',
475  # print 'Free space: %d MB. Need: %d MB. Exiting...\n' % (free_space / 1024, size_needed /1048576)
476  # exit()
477  # elif size_needed:
478  # print 'Free space on disk: %d MB.\n' % (free_space / 1024,)
479 
def check_disk_for_space
Definition: utils_v2.py:462
def utils_v2.get_relval_cmssw_version (   file)

Definition at line 92 of file utils_v2.py.

Referenced by get_version().

92 
93 def get_relval_cmssw_version(file):
94  cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
95  gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-([\w\d]*)_V\d*\w?(_[\w\d]*)?-v', file)
96  if cmssw_release and gr_r_version:
97  return (cmssw_release[0], gr_r_version[0])
def get_relval_cmssw_version
Definition: utils_v2.py:92
def utils_v2.get_relval_id (   file)
Returns unique relval ID (dataset name) for a given file.

Definition at line 98 of file utils_v2.py.

98 
99 def get_relval_id(file):
100  """Returns unique relval ID (dataset name) for a given file."""
101  dataset_name = re.findall('R\d{9}__([\w\d]*)__CMSSW_', file)
102  return dataset_name[0]
def get_relval_id
Definition: utils_v2.py:98
def utils_v2.get_relval_max_version (   files)
Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 80 of file utils_v2.py.

References get_relval_version().

80 
81 def get_relval_max_version(files):
82  """Returns file with maximum version at a) beggining of the file,
83  e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
84  max_file = files[0]
85  max_v = get_relval_version(files[0])
86  for file in files:
87  file_v = get_relval_version(file)
88  if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
89  max_file = file
90  max_v = file_v
91  return max_file
def get_relval_version
-------------—— Make files pairs: RelVal utils ---------------——
Definition: utils_v2.py:73
def get_relval_max_version
Definition: utils_v2.py:80
def utils_v2.get_relval_version (   file)

-------------—— Make files pairs: RelVal utils ---------------——

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 73 of file utils_v2.py.

Referenced by get_relval_max_version().

73 
74 def get_relval_version(file):
75  """Returns tuple (CMSSW version, run version) for specified file."""
76  cmssw_version = re.findall('DQM_V(\d*)_', file)
77  run_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-[\w\d]*_V\d*\w?(?:_[\w\d]*)?-v(\d*)__', file)
78  if cmssw_version and run_version:
79  return (int(cmssw_version[0]), int(run_version[0]))
def get_relval_version
-------------—— Make files pairs: RelVal utils ---------------——
Definition: utils_v2.py:73
def utils_v2.get_relvaldata_cmssw_version (   file)
Returns tuple (CMSSW release, GR_R version) for specified RelValData file.

Definition at line 42 of file utils_v2.py.

Referenced by get_version().

42 
44  """Returns tuple (CMSSW release, GR_R version) for specified RelValData file."""
45  cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
46  gr_r_version = re.findall('-(GR_R_\d*_V\d*\w?)(?:_RelVal)?_', file)
47  if not gr_r_version:
48  gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-(\w*)_RelVal_', file)
49  if cmssw_release and gr_r_version:
50  return (cmssw_release[0], gr_r_version[0])
def get_relvaldata_cmssw_version
Definition: utils_v2.py:42
def utils_v2.get_relvaldata_id (   file)

-----------—— Make files pairs: RelValData utils --------------——

Returns unique relvaldata ID for a given file.

Definition at line 32 of file utils_v2.py.

32 
33 def get_relvaldata_id(file):
34  """Returns unique relvaldata ID for a given file."""
35  run_id = re.search('R\d{9}', file)
36  run = re.search('_RelVal_([\w\d]*)-v\d__', file)
37  if not run:
38  run = re.search('GR_R_\d*_V\d*C?_([\w\d]*)-v\d__', file)
39  if run_id and run:
40  return (run_id.group(), run.group(1))
41  return None
def get_relvaldata_id
-----------—— Make files pairs: RelValData utils --------------——
Definition: utils_v2.py:32
def utils_v2.get_relvaldata_max_version (   files)
Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 60 of file utils_v2.py.

References get_relvaldata_version().

60 
62  """Returns file with maximum version at a) beggining of the file,
63  e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
64  max_file = files[0]
65  max_v = get_relvaldata_version(files[0])
66  for file in files:
67  file_v = get_relvaldata_version(file)
68  if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
69  max_file = file
70  max_v = file_v
71  return max_file
def get_relvaldata_version
Definition: utils_v2.py:51
def get_relvaldata_max_version
Definition: utils_v2.py:60
def utils_v2.get_relvaldata_version (   file)
Returns tuple (CMSSW version, run version) for specified file.

Definition at line 51 of file utils_v2.py.

Referenced by get_relvaldata_max_version().

51 
52 def get_relvaldata_version(file):
53  """Returns tuple (CMSSW version, run version) for specified file."""
54  cmssw_version = re.findall('DQM_V(\d*)_', file)
55  run_version = re.findall('_RelVal_[\w\d]*-v(\d)__', file)
56  if not run_version:
57  run_version = re.findall('GR_R_\d*_V\d*C?_[\w\d]*-v(\d)__', file)
58  if cmssw_version and run_version:
59  return (int(cmssw_version[0]), int(run_version[0]))
def get_relvaldata_version
Definition: utils_v2.py:51
def utils_v2.get_size_to_download (   work_path,
  files_with_urls 
)
Returns file list to download and total size to download.

Definition at line 446 of file utils_v2.py.

References join(), and print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

447 def get_size_to_download(work_path, files_with_urls):
448  """Returns file list to download and total size to download."""
449  opener = build_opener(X509CertOpen())
450  size_to_download = 0
451  files_to_download = []
452  for filename, url in files_with_urls:
453  url_file = opener.open(Request(url))
454  size = int(url_file.headers["Content-Length"])
455  file_path = join(work_path, filename)
456  if exists(file_path) and getsize(file_path) / 1024 == size / 1024:
457  print("Exists on disk %s." % filename)
458  else:
459  size_to_download += size
460  files_to_download.append(url)
461  return size_to_download, files_to_download
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def get_size_to_download
Definition: utils_v2.py:446
def utils_v2.get_version (   filename)
Returns CMSSW and GR_R versions for the given filename.

Definition at line 435 of file utils_v2.py.

References get_relval_cmssw_version(), get_relvaldata_cmssw_version(), is_relvaldata(), join(), and digitizers_cfi.strip.

436 def get_version(filename):
437  """Returns CMSSW and GR_R versions for the given filename."""
438  if is_relvaldata([filename]):
439  version_elems = get_relvaldata_cmssw_version(filename)
440  else:
441  relval_version = get_relval_cmssw_version(filename)
442  version_elems = (relval_version[0], relval_version[1][0], relval_version[1][1])
443  version_elems = [elem.strip('_').strip('RelVal_') for elem in version_elems]
444  return '___'.join([elem for elem in version_elems if elem])
445 
def get_relvaldata_cmssw_version
Definition: utils_v2.py:42
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def get_relval_cmssw_version
Definition: utils_v2.py:92
def is_relvaldata
--------------------— Make file pairs -----------------------—
Definition: utils_v2.py:104
def get_version
Definition: utils_v2.py:435
def utils_v2.init_database (   db_path)

Utils.

Definition at line 391 of file utils_v2.py.

References print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare(), and compare_using_files_v2.RootFileComparison.was_compared().

392 def init_database(db_path):
393  print('Initialising DB: %s...' % basename(db_path), end=' ')
394  conn = sqlite3.connect(db_path)
395 
396  ## Creates tables
397  c = conn.cursor()
398  c.execute("""CREATE TABLE IF NOT EXISTS ReleaseComparison (
399  id INTEGER PRIMARY KEY,
400  title TEXT,
401  release1 TEXT,
402  release2 TEXT,
403  statistical_test TEXT
404  );""")
405  c.execute("""CREATE TABLE IF NOT EXISTS Directory (
406  id INTEGER PRIMARY KEY,
407  name TEXT,
408  parent_id INTEGER,
409  from_histogram_id INTEGER,
410  till_histogram_id INTEGER,
411  FOREIGN KEY (parent_id) REFERENCES Directory(id)
412  FOREIGN KEY (from_histogram_id) REFERENCES HistogramComparison(id)
413  FOREIGN KEY (till_histogram_id) REFERENCES HistogramComparison(id)
414  )""")
415  c.execute("""CREATE TABLE IF NOT EXISTS RootFileComparison (
416  id INTEGER PRIMARY KEY,
417  filename1 TEXT,
418  filename2 TEXT,
419  release_comparison_id INTEGER,
420  directory_id INTEGER,
421  FOREIGN KEY (release_comparison_id) REFERENCES ReleaseComparison(id),
422  FOREIGN KEY (directory_id) REFERENCES Directory(id)
423  )""")
424  c.execute("""CREATE TABLE IF NOT EXISTS HistogramComparison (
425  id INTEGER PRIMARY KEY,
426  name TEXT,
427  p_value REAL,
428  directory_id INTEGER,
429  FOREIGN KEY (directory_id) REFERENCES Directory(id)
430  )""")
431 
432  print('Done.')
433  return db_path
434 
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def init_database
Utils.
Definition: utils_v2.py:391
def utils_v2.is_relvaldata (   files)

--------------------— Make file pairs -----------------------—

Definition at line 104 of file utils_v2.py.

References any().

Referenced by get_version(), and make_file_pairs().

105 def is_relvaldata(files):
106  is_relvaldata_re = re.compile('_RelVal_')
107  return any([is_relvaldata_re.search(filename) for filename in files])
bool any(const std::vector< T > &v, const T &what)
Definition: ECalSD.cc:37
def is_relvaldata
--------------------— Make file pairs -----------------------—
Definition: utils_v2.py:104
def utils_v2.make_file_pairs (   files1,
  files2 
)

Definition at line 108 of file utils_v2.py.

References bitset_utilities.append(), beamvalidation.exit(), is_relvaldata(), print(), and str.

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

109 def make_file_pairs(files1, files2):
110  print('\n################# Analyzing files ###################')
111  ## Select functions to use
112  if is_relvaldata(files1):
113  is_relval_data = True
114  get_cmssw_version = get_relvaldata_cmssw_version
115  get_id = get_relvaldata_id
116  get_max_version = get_relvaldata_max_version
117  else:
118  is_relval_data = False
119  get_cmssw_version = get_relval_cmssw_version
120  get_id = get_relval_id
121  get_max_version = get_relval_max_version
122 
123  ## Divide files into groups
124  versions1, versions2 = dict(), dict() # {version1: [file1, file2, ...], version2: [...], ...}
125  for files, versions in (files1, versions1), (files2, versions2):
126  for file in files:
127  version = get_cmssw_version(file)
128  if version:
129  if version in versions:
130  versions[version].append(file)
131  else:
132  versions[version] = [file]
133 
134  ## Print the division into groups
135  print('For RELEASE1 found file groups:')
136  for version in versions1:
137  print(' %s: %d files' % (str(version), len(versions1[version])))
138  if not versions1:
139  print('None.')
140 
141  print('\nFor RELEASE2 found file groups:')
142  for version in versions2:
143  print(' %s: %d files' % (str(version), len(versions2[version])))
144  if not versions2:
145  print('None.')
146 
147  if not len(versions1) or not len(versions2):
148  print('\nNot enough file groups. Exiting...\n')
149  exit()
150 
151  ## Pair till you find pairs.
152  pairs = []
153  for v1 in sorted(versions1, key=lambda x: len(versions1[x]), reverse=True):
154  for v2 in sorted(versions2, key=lambda x: len(versions2[x]), reverse=True):
155  if v1 == v2:
156  continue
157  ## Print the groups.
158  print('\n################# Pairing the files ###################')
159  print('%s (%d files) VS %s (%d files):\n' % (str(v1),
160  len(versions1[v1]), str(v2), len(versions2[v2])))
161 
162  ## Pairing two versions
163  for unique_id in set([get_id(file) for file in versions1[v1]]):
164  if is_relval_data:
165  dataset_re = re.compile(unique_id[0] + '_')
166  run_re = re.compile(unique_id[1])
167  c1_files = [file for file in versions1[v1] if dataset_re.search(file) and run_re.search(file)]
168  c2_files = [file for file in versions2[v2] if dataset_re.search(file) and run_re.search(file)]
169  else:
170  dataset_re = re.compile(unique_id + '_')
171  c1_files = [file for file in versions1[v1] if dataset_re.search(file)]
172  c2_files = [file for file in versions2[v2] if dataset_re.search(file)]
173 
174  if len(c1_files) > 0 and len(c2_files) > 0:
175  first_file = get_max_version(c1_files)
176  second_file = get_max_version(c2_files)
177  print('%s\n%s\n' % (first_file, second_file))
178  pairs.append((first_file, second_file))
179 
180  print("Got %d pairs." % (len(pairs)))
181  if pairs:
182  return pairs
183  print('Found no file pairs. Exiting..\n')
184  exit()
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def is_relvaldata
--------------------— Make file pairs -----------------------—
Definition: utils_v2.py:104
def make_file_pairs
Definition: utils_v2.py:108
#define str(s)
def utils_v2.recursive_search_online (   url,
  rel1,
  frags1,
  rel2,
  frags2 
)
Recursively searches for files, that matches the pattern.

Definition at line 216 of file utils_v2.py.

References python.cmstools.all(), auth_wget(), join(), and print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

217 def recursive_search_online(url, rel1, frags1, rel2, frags2):
218  """Recursively searches for files, that matches the pattern."""
219  if not url:
220  url = 'https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelValData/'
221  g1, g2 = recursive_search_online(url, rel1, frags1, rel2, frags2)
222  url = 'https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/'
223  g3, g4 = recursive_search_online(url, rel1, frags1, rel2, frags2)
224  g1.update(g3), g2.update(g4)
225  return g1, g2
226 
227  domain = '://'.join(urlparse(url)[:2])
228 
229  ## Compile regular expressions
230  href_re = re.compile(r"<a href='([-./\w]*)'>([-./\w]*)<")
231 
232  def compile_res(rel, frags):
233  frags = frags.split(',')
234  regexps = [s for s in frags if not s.startswith('!')]
235  regexps += ['^((?%s).)*$' % s for s in frags if s.startswith('!')]
236  regexps += [rel + '-', '.root']
237  return [re.compile(r) for r in regexps]
238 
239  res1 = compile_res(rel1, frags1)
240  res2 = compile_res(rel2, frags2)
241 
242  ## Recursively find files that matches regular expressions
243  hrefs = [(name, path) for path, name in href_re.findall(auth_wget(url))[1:]]
244  files_with_urls1, files_with_urls2 = dict(), dict()
245  for name, path in hrefs:
246  if splitext(name)[1]: # If file
247  if all([r.search(name) for r in res1]):
248  files_with_urls1[name] = domain + path
249  if all([r.search(name) for r in res2]):
250  files_with_urls2[name] = domain + path
251  else:
252  print(domain + path)
253  new_hrefs = href_re.findall(auth_wget(domain + path))[1:]
254  hrefs.extend([(name, path) for path, name in new_hrefs])
255  return files_with_urls1, files_with_urls2
def auth_wget
-----------------— Recursife file downloader --------------------—
Definition: utils_v2.py:186
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def all
workaround iterator generators for ROOT classes
Definition: cmstools.py:25
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def recursive_search_online
Definition: utils_v2.py:216
def utils_v2.search_on_disk (   work_path,
  rel1,
  frags1,
  rel2,
  frags2 
)

Definition at line 256 of file utils_v2.py.

References python.cmstools.all(), beamvalidation.exit(), and print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

257 def search_on_disk(work_path, rel1, frags1, rel2, frags2):
258  if not work_path:
259  print('No working directory specified. Use "--dir DIR" option to ' +\
260  'specify working directory. Exiting...')
261  exit()
262  ## Compile regular expressions
263  def compile_res(rel, frags):
264  frags = frags.split(',')
265  regexps = [s for s in frags if not s.startswith('!')]
266  regexps += ['^((?%s).)*$' % s for s in frags if s.startswith('!')]
267  regexps += [rel + '-', '.root']
268  return [re.compile(r) for r in regexps]
269 
270  res1 = compile_res(rel1, frags1)
271  res2 = compile_res(rel2, frags2)
272 
273  ## Recursively find files that matches regular expressions
274  files = listdir(work_path)
275  files1, files2 = [], []
276  for name in files:
277  if splitext(name)[1]:
278  if all([r.search(name) for r in res1]):
279  files1.append(name)
280  if all([r.search(name) for r in res2]):
281  files2.append(name)
282  return files1, files2
283 
def search_on_disk
Definition: utils_v2.py:256
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def all
workaround iterator generators for ROOT classes
Definition: cmstools.py:25
def utils_v2.show_status_bar (   total_size)
Shows download status.

Definition at line 480 of file utils_v2.py.

481 def show_status_bar(total_size):
482  """Shows download status."""
483  q = show_status_bar.q
484  total_size = total_size / (1024*1024)
485  downloaded = 0
486  while downloaded < total_size:
487  try:
488  o = q.get(timeout=20)
489  downloaded += 1
490  print('\r %d/%d MB %d%% ' % (downloaded, total_size, 100*downloaded/total_size), end=' ')
491  sys.stdout.flush()
492  except Empty:
493  time.sleep(1)
494  break
def show_status_bar
Definition: utils_v2.py:480
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

Variable Documentation

dictionary utils_v2.comparison_errors
Initial value:
1 = {
2  'Missing histogram': -1,
3  'Histograms have different types': -2,
4  'Object is not a histogram': -3,
5  'Ranges of histograms are different': -4
6  }

Exception definitions.

Definition at line 285 of file utils_v2.py.

dictionary utils_v2.tests = {KolmogorovTest.name: KolmogorovTest, Chi2Test.name: Chi2Test}

Definition at line 388 of file utils_v2.py.