CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
cmsswVersionTools.py
Go to the documentation of this file.
1 import FWCore.ParameterSet.Config as cms
2 
3 from FWCore.GuiBrowsers.ConfigToolBase import *
4 from PhysicsTools.PatAlgos.tools.helpers import *
5 from PhysicsTools.PatAlgos.tools.jetTools import *
6 from Configuration.AlCa.autoCond import autoCond
7 import Utilities.General.cmssw_das_client as das_client
8 
9 import os
10 import socket
11 
12 ## ------------------------------------------------------
13 ## Automatic pick-up of RelVal input files
14 ## ------------------------------------------------------
15 
16 class PickRelValInputFiles( ConfigToolBase ):
17  """ Picks up RelVal input files automatically and
18  returns a vector of strings with the paths to be used in [PoolSource].fileNames
19  PickRelValInputFiles( cmsswVersion, relVal, dataTier, condition, globalTag, maxVersions, skipFiles, numberOfFiles, debug )
20  - useDAS : switch to perform query in DAS rather than in DBS
21  optional; default: False
22  - cmsswVersion : CMSSW release to pick up the RelVal files from
23  optional; default: the current release (determined automatically from environment)
24  - formerVersion: use the last before the last valid CMSSW release to pick up the RelVal files from
25  applies also, if 'cmsswVersion' is set explicitly
26  optional; default: False
27  - relVal : RelVal sample to be used
28  optional; default: 'RelValTTbar'
29  - dataTier : data tier to be used
30  optional; default: 'GEN-SIM-RECO'
31  - condition : identifier of GlobalTag as defined in Configurations/PyReleaseValidation/python/autoCond.py
32  possibly overwritten, if 'globalTag' is set explicitly
33  optional; default: 'startup'
34  - globalTag : name of GlobalTag as it is used in the data path of the RelVals
35  optional; default: determined automatically as defined by 'condition' in Configurations/PyReleaseValidation/python/autoCond.py
36  !!! Determination is done for the release one runs in, not for the release the RelVals have been produced in.
37  !!! Example of deviation: data RelVals (CMSSW_4_1_X) might not only have the pure name of the GlobalTag 'GR_R_311_V2' in the full path,
38  but also an extension identifying the data: 'GR_R_311_V2_RelVal_wzMu2010B'
39  - maxVersions : max. versioning number of RelVal to check
40  optional; default: 9
41  - skipFiles : number of files to skip for a found RelVal sample
42  optional; default: 0
43  - numberOfFiles: number of files to pick up
44  setting it to negative values, returns all found ('skipFiles' remains active though)
45  optional; default: -1
46  - debug : switch to enable enhanced messages in 'stdout'
47  optional; default: False
48  """
49 
50  _label = 'pickRelValInputFiles'
51  _defaultParameters = dicttypes.SortedKeysDict()
52 
53  def getDefaultParameters( self ):
54  return self._defaultParameters
55 
56  def __init__( self ):
57  ConfigToolBase.__init__( self )
58  self.addParameter( self._defaultParameters, 'useDAS' , False , '' )
59  self.addParameter( self._defaultParameters, 'cmsswVersion' , os.getenv( "CMSSW_VERSION" ) , 'auto from environment' )
60  self.addParameter( self._defaultParameters, 'formerVersion', False , '' )
61  self.addParameter( self._defaultParameters, 'relVal' , 'RelValTTbar' , '' )
62  self.addParameter( self._defaultParameters, 'dataTier' , 'GEN-SIM-RECO' , '' )
63  self.addParameter( self._defaultParameters, 'condition' , 'startup' , '' )
64  self.addParameter( self._defaultParameters, 'globalTag' , autoCond[ self.getDefaultParameters()[ 'condition' ].value ][ : -5 ], 'auto from \'condition\'' )
65  self.addParameter( self._defaultParameters, 'maxVersions' , 3 , '' )
66  self.addParameter( self._defaultParameters, 'skipFiles' , 0 , '' )
67  self.addParameter( self._defaultParameters, 'numberOfFiles', -1 , 'all' )
68  self.addParameter( self._defaultParameters, 'debug' , False , '' )
69  self._parameters = copy.deepcopy( self._defaultParameters )
70  self._comment = ""
71 
72  def __call__( self
73  , useDAS = None
74  , cmsswVersion = None
75  , formerVersion = None
76  , relVal = None
77  , dataTier = None
78  , condition = None
79  , globalTag = None
80  , maxVersions = None
81  , skipFiles = None
82  , numberOfFiles = None
83  , debug = None
84  ):
85  if useDAS is None:
86  useDAS = self.getDefaultParameters()[ 'useDAS' ].value
87  if cmsswVersion is None:
88  cmsswVersion = self.getDefaultParameters()[ 'cmsswVersion' ].value
89  if formerVersion is None:
90  formerVersion = self.getDefaultParameters()[ 'formerVersion' ].value
91  if relVal is None:
92  relVal = self.getDefaultParameters()[ 'relVal' ].value
93  if dataTier is None:
94  dataTier = self.getDefaultParameters()[ 'dataTier' ].value
95  if condition is None:
96  condition = self.getDefaultParameters()[ 'condition' ].value
97  if globalTag is None:
98  globalTag = autoCond[ condition ][ : -5 ] # auto from 'condition'
99  if maxVersions is None:
100  maxVersions = self.getDefaultParameters()[ 'maxVersions' ].value
101  if skipFiles is None:
102  skipFiles = self.getDefaultParameters()[ 'skipFiles' ].value
103  if numberOfFiles is None:
104  numberOfFiles = self.getDefaultParameters()[ 'numberOfFiles' ].value
105  if debug is None:
106  debug = self.getDefaultParameters()[ 'debug' ].value
107  self.setParameter( 'useDAS' , useDAS )
108  self.setParameter( 'cmsswVersion' , cmsswVersion )
109  self.setParameter( 'formerVersion', formerVersion )
110  self.setParameter( 'relVal' , relVal )
111  self.setParameter( 'dataTier' , dataTier )
112  self.setParameter( 'condition' , condition )
113  self.setParameter( 'globalTag' , globalTag )
114  self.setParameter( 'maxVersions' , maxVersions )
115  self.setParameter( 'skipFiles' , skipFiles )
116  self.setParameter( 'numberOfFiles', numberOfFiles )
117  self.setParameter( 'debug' , debug )
118  return self.apply()
119 
120  def messageEmptyList( self ):
121  print '%s DEBUG: Empty file list returned'%( self._label )
122  print ' This might be overwritten by providing input files explicitly to the source module in the main configuration file.'
123 
124  def apply( self ):
125  useDAS = self._parameters[ 'useDAS' ].value
126  cmsswVersion = self._parameters[ 'cmsswVersion' ].value
127  formerVersion = self._parameters[ 'formerVersion' ].value
128  relVal = self._parameters[ 'relVal' ].value
129  dataTier = self._parameters[ 'dataTier' ].value
130  condition = self._parameters[ 'condition' ].value # only used for GT determination in initialization, if GT not explicitly given
131  globalTag = self._parameters[ 'globalTag' ].value
132  maxVersions = self._parameters[ 'maxVersions' ].value
133  skipFiles = self._parameters[ 'skipFiles' ].value
134  numberOfFiles = self._parameters[ 'numberOfFiles' ].value
135  debug = self._parameters[ 'debug' ].value
136 
137  filePaths = []
138 
139  # Determine corresponding CMSSW version for RelVals
140  preId = '_pre'
141  patchId = '_patch' # patch releases
142  hltPatchId = '_hltpatch' # HLT patch releases
143  dqmPatchId = '_dqmpatch' # DQM patch releases
144  slhcId = '_SLHC' # SLHC releases
145  rootId = '_root' # ROOT test releases
146  ibId = '_X_' # IBs
147  if patchId in cmsswVersion:
148  cmsswVersion = cmsswVersion.split( patchId )[ 0 ]
149  elif hltPatchId in cmsswVersion:
150  cmsswVersion = cmsswVersion.split( hltPatchId )[ 0 ]
151  elif dqmPatchId in cmsswVersion:
152  cmsswVersion = cmsswVersion.split( dqmPatchId )[ 0 ]
153  elif rootId in cmsswVersion:
154  cmsswVersion = cmsswVersion.split( rootId )[ 0 ]
155  elif slhcId in cmsswVersion:
156  cmsswVersion = cmsswVersion.split( slhcId )[ 0 ]
157  elif ibId in cmsswVersion or formerVersion:
158  outputTuple = Popen( [ 'scram', 'l -c CMSSW' ], stdout = PIPE, stderr = PIPE ).communicate()
159  if len( outputTuple[ 1 ] ) != 0:
160  print '%s INFO : SCRAM error'%( self._label )
161  if debug:
162  print ' from trying to determine last valid releases before \'%s\''%( cmsswVersion )
163  print
164  print outputTuple[ 1 ]
165  print
166  self.messageEmptyList()
167  return filePaths
168  versions = { 'last' :''
169  , 'lastToLast':''
170  }
171  for line in outputTuple[ 0 ].splitlines():
172  version = line.split()[ 1 ]
173  if cmsswVersion.split( ibId )[ 0 ] in version or cmsswVersion.rpartition( '_' )[ 0 ] in version:
174  if not ( patchId in version or hltPatchId in version or dqmPatchId in version or slhcId in version or ibId in version or rootId in version ):
175  versions[ 'lastToLast' ] = versions[ 'last' ]
176  versions[ 'last' ] = version
177  if version == cmsswVersion:
178  break
179  # FIXME: ordering of output problematic ('XYZ_pre10' before 'XYZ_pre2', no "formerVersion" for 'XYZ_pre1')
180  if formerVersion:
181  # Don't use pre-releases as "former version" for other releases than CMSSW_X_Y_0
182  if preId in versions[ 'lastToLast' ] and not preId in versions[ 'last' ] and not versions[ 'last' ].endswith( '_0' ):
183  versions[ 'lastToLast' ] = versions[ 'lastToLast' ].split( preId )[ 0 ] # works only, if 'CMSSW_X_Y_0' esists ;-)
184  # Use pre-release as "former version" for CMSSW_X_Y_0
185  elif versions[ 'last' ].endswith( '_0' ) and not ( preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].startswith( versions[ 'last' ] ) ):
186  versions[ 'lastToLast' ] = ''
187  for line in outputTuple[ 0 ].splitlines():
188  version = line.split()[ 1 ]
189  versionParts = version.partition( preId )
190  if versionParts[ 0 ] == versions[ 'last' ] and versionParts[ 1 ] == preId:
191  versions[ 'lastToLast' ] = version
192  elif versions[ 'lastToLast' ] != '':
193  break
194  # Don't use CMSSW_X_Y_0 as "former version" for pre-releases
195  elif preId in versions[ 'last' ] and not preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].endswith( '_0' ):
196  versions[ 'lastToLast' ] = '' # no alternative :-(
197  cmsswVersion = versions[ 'lastToLast' ]
198  else:
199  cmsswVersion = versions[ 'last' ]
200 
201  # Debugging output
202  if debug:
203  print '%s DEBUG: Called with...'%( self._label )
204  for key in self._parameters.keys():
205  print ' %s:\t'%( key ),
206  print self._parameters[ key ].value,
207  if self._parameters[ key ].value is self.getDefaultParameters()[ key ].value:
208  print ' (default)'
209  else:
210  print
211  if key == 'cmsswVersion' and cmsswVersion != self._parameters[ key ].value:
212  if formerVersion:
213  print ' ==> modified to last to last valid release %s (s. \'formerVersion\' parameter)'%( cmsswVersion )
214  else:
215  print ' ==> modified to last valid release %s'%( cmsswVersion )
216 
217  # Check domain
218  domain = socket.getfqdn().split( '.' )
219  domainSE = ''
220  if len( domain ) == 0:
221  print '%s INFO : Cannot determine domain of this computer'%( self._label )
222  if debug:
223  self.messageEmptyList()
224  return filePaths
225  elif os.uname()[0] == "Darwin":
226  print '%s INFO : Running on MacOSX without direct access to RelVal files.'%( self._label )
227  if debug:
228  self.messageEmptyList()
229  return filePaths
230  elif len( domain ) == 1:
231  print '%s INFO : Running on local host \'%s\' without direct access to RelVal files'%( self._label, domain[ 0 ] )
232  if debug:
233  self.messageEmptyList()
234  return filePaths
235  if not ( ( domain[ -2 ] == 'cern' and domain[ -1 ] == 'ch' ) or ( domain[ -2 ] == 'fnal' and domain[ -1 ] == 'gov' ) ):
236  print '%s INFO : Running on site \'%s.%s\' without direct access to RelVal files'%( self._label, domain[ -2 ], domain[ -1 ] )
237  if debug:
238  self.messageEmptyList()
239  return filePaths
240  if domain[ -2 ] == 'cern':
241  domainSE = 'T2_CH_CERN'
242  elif domain[ -2 ] == 'fnal':
243  domainSE = 'T1_US_FNAL_MSS'
244  if debug:
245  print '%s DEBUG: Running at site \'%s.%s\''%( self._label, domain[ -2 ], domain[ -1 ] )
246  print '%s DEBUG: Looking for SE \'%s\''%( self._label, domainSE )
247 
248  # Find files
249  validVersion = 0
250  dataset = ''
251  datasetAll = '/%s/%s-%s-v*/%s'%( relVal, cmsswVersion, globalTag, dataTier )
252  if useDAS:
253  if debug:
254  print '%s DEBUG: Using DAS query'%( self._label )
255  dasLimit = numberOfFiles
256  if dasLimit <= 0:
257  dasLimit += 1
258  for version in range( maxVersions, 0, -1 ):
259  filePaths = []
260  filePathsTmp = []
261  fileCount = 0
262  dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
263  dasQuery = 'file dataset=%s | grep file.name'%( dataset )
264  if debug:
265  print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
266  print ' \'%s\''%( dasQuery )
267  # partially stolen from das_client.py for option '--format=plain', needs filter ("grep") in the query
268  jsondict = das_client.get_data(dasQuery, dasLimit)
269  if debug:
270  print '%s DEBUG: Received DAS JSON dictionary:'%( self._label )
271  print ' \'%s\''%( jsondict )
272  if jsondict[ 'status' ] != 'ok':
273  print 'There was a problem while querying DAS with query \'%s\'. Server reply was:\n %s' % (dasQuery, jsondict)
274  exit( 1 )
275  mongo_query = jsondict[ 'mongo_query' ]
276  filters = mongo_query[ 'filters' ]
277  data = jsondict[ 'data' ]
278  if debug:
279  print '%s DEBUG: Query in JSON dictionary:'%( self._label )
280  print ' \'%s\''%( mongo_query )
281  print '%s DEBUG: Filters in query:'%( self._label )
282  print ' \'%s\''%( filters )
283  print '%s DEBUG: Data in JSON dictionary:'%( self._label )
284  print ' \'%s\''%( data )
285  for row in data:
286  filePath = [ r for r in das_client.get_value( row, filters[ 'grep' ] ) ][ 0 ]
287  if debug:
288  print '%s DEBUG: Testing file entry \'%s\''%( self._label, filePath )
289  if len( filePath ) > 0:
290  if validVersion != version:
291  jsontestdict = das_client.get_data('site dataset=%s | grep site.name'%( dataset ), 999)
292  mongo_testquery = jsontestdict[ 'mongo_query' ]
293  testfilters = mongo_testquery[ 'filters' ]
294  testdata = jsontestdict[ 'data' ]
295  if debug:
296  print '%s DEBUG: Received DAS JSON dictionary (site test):'%( self._label )
297  print ' \'%s\''%( jsontestdict )
298  print '%s DEBUG: Query in JSON dictionary (site test):'%( self._label )
299  print ' \'%s\''%( mongo_testquery )
300  print '%s DEBUG: Filters in query (site test):'%( self._label )
301  print ' \'%s\''%( testfilters )
302  print '%s DEBUG: Data in JSON dictionary (site test):'%( self._label )
303  print ' \'%s\''%( testdata )
304  foundSE = False
305  for testrow in testdata:
306  siteName = [ tr for tr in das_client.get_value( testrow, testfilters[ 'grep' ] ) ][ 0 ]
307  if siteName == domainSE:
308  foundSE = True
309  break
310  if not foundSE:
311  if debug:
312  print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
313  break
314  validVersion = version
315  if debug:
316  print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
317  if numberOfFiles == 0:
318  break
319  # protect from double entries ( 'unique' flag in query does not work here)
320  if not filePath in filePathsTmp:
321  filePathsTmp.append( filePath )
322  if debug:
323  print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
324  fileCount += 1
325  # needed, since and "limit" overrides "idx" in 'get_data' (==> "idx" set to '0' rather than "skipFiles")
326  if fileCount > skipFiles:
327  filePaths.append( filePath )
328  elif debug:
329  print '%s DEBUG: File \'%s\' found again'%( self._label, filePath )
330  if validVersion > 0:
331  if numberOfFiles == 0 and debug:
332  print '%s DEBUG: No files requested'%( self._label )
333  break
334  else:
335  if debug:
336  print '%s DEBUG: Using DBS query'%( self._label )
337  print '%s WARNING: DBS query disabled for DBS3 transition to new API'%( self._label )
338  #for version in range( maxVersions, 0, -1 ):
339  #filePaths = []
340  #fileCount = 0
341  #dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
342  #dbsQuery = 'find file where dataset = %s'%( dataset )
343  #if debug:
344  #print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
345  #print ' \'%s\''%( dbsQuery )
346  #foundSE = False
347  #for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ).readlines():
348  #if line.find( '.root' ) != -1:
349  #if validVersion != version:
350  #if not foundSE:
351  #dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
352  #if debug:
353  #print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
354  #print ' \'%s\''%( dbsSiteQuery )
355  #for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ).readlines():
356  #if lineSite.find( dataset ) != -1:
357  #foundSE = True
358  #break
359  #if not foundSE:
360  #if debug:
361  #print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
362  #break
363  #validVersion = version
364  #if debug:
365  #print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
366  #if numberOfFiles == 0:
367  #break
368  #filePath = line.replace( '\n', '' )
369  #if debug:
370  #print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
371  #fileCount += 1
372  #if fileCount > skipFiles:
373  #filePaths.append( filePath )
374  #if not numberOfFiles < 0:
375  #if numberOfFiles <= len( filePaths ):
376  #break
377  #if validVersion > 0:
378  #if numberOfFiles == 0 and debug:
379  #print '%s DEBUG: No files requested'%( self._label )
380  #break
381 
382  # Check output and return
383  if validVersion == 0:
384  print '%s WARNING : No RelVal file(s) found at all in datasets \'%s*\' on SE \'%s\''%( self._label, datasetAll, domainSE )
385  if debug:
386  self.messageEmptyList()
387  elif len( filePaths ) == 0:
388  print '%s WARNING : No RelVal file(s) picked up in dataset \'%s\''%( self._label, dataset )
389  if debug:
390  self.messageEmptyList()
391  elif len( filePaths ) < numberOfFiles:
392  print '%s INFO : Only %i RelVal file(s) instead of %i picked up in dataset \'%s\''%( self._label, len( filePaths ), numberOfFiles, dataset )
393 
394  if debug:
395  print '%s DEBUG: returning %i file(s):\n%s'%( self._label, len( filePaths ), filePaths )
396  return filePaths
397 
398 pickRelValInputFiles = PickRelValInputFiles()
communicate
static void * communicate(void *obj)
Definition: DQMNet.cc:1246
cmsswVersionTools.PickRelValInputFiles
Automatic pick-up of RelVal input files
Definition: cmsswVersionTools.py:16
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5