CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
cmsswVersionTools.py
Go to the documentation of this file.
1 import FWCore.ParameterSet.Config as cms
2 
3 from FWCore.GuiBrowsers.ConfigToolBase import *
4 from PhysicsTools.PatAlgos.tools.helpers import *
5 from PhysicsTools.PatAlgos.tools.jetTools import *
6 from Configuration.AlCa.autoCond import autoCond
7 
8 import os
9 import socket
10 from subprocess import *
11 import json
12 import das_client
13 
14 
15 ## ------------------------------------------------------
16 ## Automatic pick-up of RelVal input files
17 ## ------------------------------------------------------
18 
19 class PickRelValInputFiles( ConfigToolBase ):
20  """ Picks up RelVal input files automatically and
21  returns a vector of strings with the paths to be used in [PoolSource].fileNames
22  PickRelValInputFiles( cmsswVersion, relVal, dataTier, condition, globalTag, maxVersions, skipFiles, numberOfFiles, debug )
23  - useDAS : switch to perform query in DAS rather than in DBS
24  optional; default: False
25  - cmsswVersion : CMSSW release to pick up the RelVal files from
26  optional; default: the current release (determined automatically from environment)
27  - formerVersion: use the last before the last valid CMSSW release to pick up the RelVal files from
28  applies also, if 'cmsswVersion' is set explicitly
29  optional; default: False
30  - relVal : RelVal sample to be used
31  optional; default: 'RelValTTbar'
32  - dataTier : data tier to be used
33  optional; default: 'GEN-SIM-RECO'
34  - condition : identifier of GlobalTag as defined in Configurations/PyReleaseValidation/python/autoCond.py
35  possibly overwritten, if 'globalTag' is set explicitly
36  optional; default: 'startup'
37  - globalTag : name of GlobalTag as it is used in the data path of the RelVals
38  optional; default: determined automatically as defined by 'condition' in Configurations/PyReleaseValidation/python/autoCond.py
39  !!! Determination is done for the release one runs in, not for the release the RelVals have been produced in.
40  !!! Example of deviation: data RelVals (CMSSW_4_1_X) might not only have the pure name of the GlobalTag 'GR_R_311_V2' in the full path,
41  but also an extension identifying the data: 'GR_R_311_V2_RelVal_wzMu2010B'
42  - maxVersions : max. versioning number of RelVal to check
43  optional; default: 9
44  - skipFiles : number of files to skip for a found RelVal sample
45  optional; default: 0
46  - numberOfFiles: number of files to pick up
47  setting it to negative values, returns all found ('skipFiles' remains active though)
48  optional; default: -1
49  - debug : switch to enable enhanced messages in 'stdout'
50  optional; default: False
51  """
52 
53  _label = 'pickRelValInputFiles'
54  _defaultParameters = dicttypes.SortedKeysDict()
55 
56  def getDefaultParameters( self ):
57  return self._defaultParameters
58 
59  def __init__( self ):
60  ConfigToolBase.__init__( self )
61  self.addParameter( self._defaultParameters, 'useDAS' , False , '' )
62  self.addParameter( self._defaultParameters, 'cmsswVersion' , os.getenv( "CMSSW_VERSION" ) , 'auto from environment' )
63  self.addParameter( self._defaultParameters, 'formerVersion', False , '' )
64  self.addParameter( self._defaultParameters, 'relVal' , 'RelValTTbar' , '' )
65  self.addParameter( self._defaultParameters, 'dataTier' , 'GEN-SIM-RECO' , '' )
66  self.addParameter( self._defaultParameters, 'condition' , 'startup' , '' )
67  self.addParameter( self._defaultParameters, 'globalTag' , autoCond[ self.getDefaultParameters()[ 'condition' ].value ][ : -5 ], 'auto from \'condition\'' )
68  self.addParameter( self._defaultParameters, 'maxVersions' , 3 , '' )
69  self.addParameter( self._defaultParameters, 'skipFiles' , 0 , '' )
70  self.addParameter( self._defaultParameters, 'numberOfFiles', -1 , 'all' )
71  self.addParameter( self._defaultParameters, 'debug' , False , '' )
72  self._parameters = copy.deepcopy( self._defaultParameters )
73  self._comment = ""
74 
75  def __call__( self
76  , useDAS = None
77  , cmsswVersion = None
78  , formerVersion = None
79  , relVal = None
80  , dataTier = None
81  , condition = None
82  , globalTag = None
83  , maxVersions = None
84  , skipFiles = None
85  , numberOfFiles = None
86  , debug = None
87  ):
88  if useDAS is None:
89  useDAS = self.getDefaultParameters()[ 'useDAS' ].value
90  if cmsswVersion is None:
91  cmsswVersion = self.getDefaultParameters()[ 'cmsswVersion' ].value
92  if formerVersion is None:
93  formerVersion = self.getDefaultParameters()[ 'formerVersion' ].value
94  if relVal is None:
95  relVal = self.getDefaultParameters()[ 'relVal' ].value
96  if dataTier is None:
97  dataTier = self.getDefaultParameters()[ 'dataTier' ].value
98  if condition is None:
99  condition = self.getDefaultParameters()[ 'condition' ].value
100  if globalTag is None:
101  globalTag = autoCond[ condition ][ : -5 ] # auto from 'condition'
102  if maxVersions is None:
103  maxVersions = self.getDefaultParameters()[ 'maxVersions' ].value
104  if skipFiles is None:
105  skipFiles = self.getDefaultParameters()[ 'skipFiles' ].value
106  if numberOfFiles is None:
107  numberOfFiles = self.getDefaultParameters()[ 'numberOfFiles' ].value
108  if debug is None:
109  debug = self.getDefaultParameters()[ 'debug' ].value
110  self.setParameter( 'useDAS' , useDAS )
111  self.setParameter( 'cmsswVersion' , cmsswVersion )
112  self.setParameter( 'formerVersion', formerVersion )
113  self.setParameter( 'relVal' , relVal )
114  self.setParameter( 'dataTier' , dataTier )
115  self.setParameter( 'condition' , condition )
116  self.setParameter( 'globalTag' , globalTag )
117  self.setParameter( 'maxVersions' , maxVersions )
118  self.setParameter( 'skipFiles' , skipFiles )
119  self.setParameter( 'numberOfFiles', numberOfFiles )
120  self.setParameter( 'debug' , debug )
121  return self.apply()
122 
123  def messageEmptyList( self ):
124  print '%s DEBUG: Empty file list returned'%( self._label )
125  print ' This might be overwritten by providing input files explicitly to the source module in the main configuration file.'
126 
127  def apply( self ):
128  useDAS = self._parameters[ 'useDAS' ].value
129  cmsswVersion = self._parameters[ 'cmsswVersion' ].value
130  formerVersion = self._parameters[ 'formerVersion' ].value
131  relVal = self._parameters[ 'relVal' ].value
132  dataTier = self._parameters[ 'dataTier' ].value
133  condition = self._parameters[ 'condition' ].value # only used for GT determination in initialization, if GT not explicitly given
134  globalTag = self._parameters[ 'globalTag' ].value
135  maxVersions = self._parameters[ 'maxVersions' ].value
136  skipFiles = self._parameters[ 'skipFiles' ].value
137  numberOfFiles = self._parameters[ 'numberOfFiles' ].value
138  debug = self._parameters[ 'debug' ].value
139 
140  filePaths = []
141 
142  # Determine corresponding CMSSW version for RelVals
143  preId = '_pre'
144  patchId = '_patch' # patch releases
145  hltPatchId = '_hltpatch' # HLT patch releases
146  dqmPatchId = '_dqmpatch' # DQM patch releases
147  slhcId = '_SLHC' # SLHC releases
148  rootId = '_root' # ROOT test releases
149  ibId = '_X_' # IBs
150  if patchId in cmsswVersion:
151  cmsswVersion = cmsswVersion.split( patchId )[ 0 ]
152  elif hltPatchId in cmsswVersion:
153  cmsswVersion = cmsswVersion.split( hltPatchId )[ 0 ]
154  elif dqmPatchId in cmsswVersion:
155  cmsswVersion = cmsswVersion.split( dqmPatchId )[ 0 ]
156  elif rootId in cmsswVersion:
157  cmsswVersion = cmsswVersion.split( rootId )[ 0 ]
158  elif slhcId in cmsswVersion:
159  cmsswVersion = cmsswVersion.split( slhcId )[ 0 ]
160  elif ibId in cmsswVersion or formerVersion:
161  outputTuple = Popen( [ 'scram', 'l -c CMSSW' ], stdout = PIPE, stderr = PIPE ).communicate()
162  if len( outputTuple[ 1 ] ) != 0:
163  print '%s INFO : SCRAM error'%( self._label )
164  if debug:
165  print ' from trying to determine last valid releases before \'%s\''%( cmsswVersion )
166  print
167  print outputTuple[ 1 ]
168  print
169  self.messageEmptyList()
170  return filePaths
171  versions = { 'last' :''
172  , 'lastToLast':''
173  }
174  for line in outputTuple[ 0 ].splitlines():
175  version = line.split()[ 1 ]
176  if cmsswVersion.split( ibId )[ 0 ] in version or cmsswVersion.rpartition( '_' )[ 0 ] in version:
177  if not ( patchId in version or hltPatchId in version or dqmPatchId in version or slhcId in version or ibId in version or rootId in version ):
178  versions[ 'lastToLast' ] = versions[ 'last' ]
179  versions[ 'last' ] = version
180  if version == cmsswVersion:
181  break
182  # FIXME: ordering of output problematic ('XYZ_pre10' before 'XYZ_pre2', no "formerVersion" for 'XYZ_pre1')
183  if formerVersion:
184  # Don't use pre-releases as "former version" for other releases than CMSSW_X_Y_0
185  if preId in versions[ 'lastToLast' ] and not preId in versions[ 'last' ] and not versions[ 'last' ].endswith( '_0' ):
186  versions[ 'lastToLast' ] = versions[ 'lastToLast' ].split( preId )[ 0 ] # works only, if 'CMSSW_X_Y_0' esists ;-)
187  # Use pre-release as "former version" for CMSSW_X_Y_0
188  elif versions[ 'last' ].endswith( '_0' ) and not ( preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].startswith( versions[ 'last' ] ) ):
189  versions[ 'lastToLast' ] = ''
190  for line in outputTuple[ 0 ].splitlines():
191  version = line.split()[ 1 ]
192  versionParts = version.partition( preId )
193  if versionParts[ 0 ] == versions[ 'last' ] and versionParts[ 1 ] == preId:
194  versions[ 'lastToLast' ] = version
195  elif versions[ 'lastToLast' ] != '':
196  break
197  # Don't use CMSSW_X_Y_0 as "former version" for pre-releases
198  elif preId in versions[ 'last' ] and not preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].endswith( '_0' ):
199  versions[ 'lastToLast' ] = '' # no alternative :-(
200  cmsswVersion = versions[ 'lastToLast' ]
201  else:
202  cmsswVersion = versions[ 'last' ]
203 
204  # Debugging output
205  if debug:
206  print '%s DEBUG: Called with...'%( self._label )
207  for key in self._parameters.keys():
208  print ' %s:\t'%( key ),
209  print self._parameters[ key ].value,
210  if self._parameters[ key ].value is self.getDefaultParameters()[ key ].value:
211  print ' (default)'
212  else:
213  print
214  if key == 'cmsswVersion' and cmsswVersion != self._parameters[ key ].value:
215  if formerVersion:
216  print ' ==> modified to last to last valid release %s (s. \'formerVersion\' parameter)'%( cmsswVersion )
217  else:
218  print ' ==> modified to last valid release %s'%( cmsswVersion )
219 
220  # Check domain
221  domain = socket.getfqdn().split( '.' )
222  domainSE = ''
223  if len( domain ) == 0:
224  print '%s INFO : Cannot determine domain of this computer'%( self._label )
225  if debug:
226  self.messageEmptyList()
227  return filePaths
228  elif os.uname()[0] == "Darwin":
229  print '%s INFO : Running on MacOSX without direct access to RelVal files.'%( self._label )
230  if debug:
231  self.messageEmptyList()
232  return filePaths
233  elif len( domain ) == 1:
234  print '%s INFO : Running on local host \'%s\' without direct access to RelVal files'%( self._label, domain[ 0 ] )
235  if debug:
236  self.messageEmptyList()
237  return filePaths
238  if not ( ( domain[ -2 ] == 'cern' and domain[ -1 ] == 'ch' ) or ( domain[ -2 ] == 'fnal' and domain[ -1 ] == 'gov' ) ):
239  print '%s INFO : Running on site \'%s.%s\' without direct access to RelVal files'%( self._label, domain[ -2 ], domain[ -1 ] )
240  if debug:
241  self.messageEmptyList()
242  return filePaths
243  if domain[ -2 ] == 'cern':
244  domainSE = 'T2_CH_CERN'
245  elif domain[ -2 ] == 'fnal':
246  domainSE = 'T1_US_FNAL_MSS'
247  if debug:
248  print '%s DEBUG: Running at site \'%s.%s\''%( self._label, domain[ -2 ], domain[ -1 ] )
249  print '%s DEBUG: Looking for SE \'%s\''%( self._label, domainSE )
250 
251  # Find files
252  validVersion = 0
253  dataset = ''
254  datasetAll = '/%s/%s-%s-v*/%s'%( relVal, cmsswVersion, globalTag, dataTier )
255  if useDAS:
256  if debug:
257  print '%s DEBUG: Using DAS query'%( self._label )
258  dasLimit = numberOfFiles
259  if dasLimit <= 0:
260  dasLimit += 1
261  for version in range( maxVersions, 0, -1 ):
262  filePaths = []
263  filePathsTmp = []
264  fileCount = 0
265  dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
266  dasQuery = 'file dataset=%s | grep file.name'%( dataset )
267  if debug:
268  print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
269  print ' \'%s\''%( dasQuery )
270  # partially stolen from das_client.py for option '--format=plain', needs filter ("grep") in the query
271  dasData = das_client.get_data( 'https://cmsweb.cern.ch', dasQuery, 0, dasLimit, False )
272  jsondict = json.loads( dasData )
273  if debug:
274  print '%s DEBUG: Received DAS data:'%( self._label )
275  print ' \'%s\''%( dasData )
276  print '%s DEBUG: Determined JSON dictionary:'%( self._label )
277  print ' \'%s\''%( jsondict )
278  if jsondict[ 'status' ] != 'ok':
279  print 'There was a problem while querying DAS with query \'%s\'. Server reply was:\n %s' % (dasQuery, dasData)
280  exit( 1 )
281  mongo_query = jsondict[ 'mongo_query' ]
282  filters = mongo_query[ 'filters' ]
283  data = jsondict[ 'data' ]
284  if debug:
285  print '%s DEBUG: Query in JSON dictionary:'%( self._label )
286  print ' \'%s\''%( mongo_query )
287  print '%s DEBUG: Filters in query:'%( self._label )
288  print ' \'%s\''%( filters )
289  print '%s DEBUG: Data in JSON dictionary:'%( self._label )
290  print ' \'%s\''%( data )
291  for row in data:
292  filePath = [ r for r in das_client.get_value( row, filters ) ][ 0 ]
293  if debug:
294  print '%s DEBUG: Testing file entry \'%s\''%( self._label, filePath )
295  if len( filePath ) > 0:
296  if validVersion != version:
297  dasTest = das_client.get_data( 'https://cmsweb.cern.ch', 'site dataset=%s | grep site.name'%( dataset ), 0, 999, False )
298  jsontestdict = json.loads( dasTest )
299  mongo_testquery = jsontestdict[ 'mongo_query' ]
300  testfilters = mongo_testquery[ 'filters' ]
301  testdata = jsontestdict[ 'data' ]
302  if debug:
303  print '%s DEBUG: Received DAS data (site test):'%( self._label )
304  print ' \'%s\''%( dasTest )
305  print '%s DEBUG: Determined JSON dictionary (site test):'%( self._label )
306  print ' \'%s\''%( jsontestdict )
307  print '%s DEBUG: Query in JSON dictionary (site test):'%( self._label )
308  print ' \'%s\''%( mongo_testquery )
309  print '%s DEBUG: Filters in query (site test):'%( self._label )
310  print ' \'%s\''%( testfilters )
311  print '%s DEBUG: Data in JSON dictionary (site test):'%( self._label )
312  print ' \'%s\''%( testdata )
313  foundSE = False
314  for testrow in testdata:
315  siteName = [ tr for tr in das_client.get_value( testrow, testfilters ) ][ 0 ]
316  if siteName == domainSE:
317  foundSE = True
318  break
319  if not foundSE:
320  if debug:
321  print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
322  break
323  validVersion = version
324  if debug:
325  print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
326  if numberOfFiles == 0:
327  break
328  # protect from double entries ( 'unique' flag in query does not work here)
329  if not filePath in filePathsTmp:
330  filePathsTmp.append( filePath )
331  if debug:
332  print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
333  fileCount += 1
334  # needed, since and "limit" overrides "idx" in 'get_data' (==> "idx" set to '0' rather than "skipFiles")
335  if fileCount > skipFiles:
336  filePaths.append( filePath )
337  elif debug:
338  print '%s DEBUG: File \'%s\' found again'%( self._label, filePath )
339  if validVersion > 0:
340  if numberOfFiles == 0 and debug:
341  print '%s DEBUG: No files requested'%( self._label )
342  break
343  else:
344  if debug:
345  print '%s DEBUG: Using DBS query'%( self._label )
346  for version in range( maxVersions, 0, -1 ):
347  filePaths = []
348  fileCount = 0
349  dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
350  dbsQuery = 'find file where dataset = %s'%( dataset )
351  if debug:
352  print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
353  print ' \'%s\''%( dbsQuery )
354  foundSE = False
355  for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ):
356  if line.find( '.root' ) != -1:
357  if validVersion != version:
358  if not foundSE:
359  dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
360  if debug:
361  print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
362  print ' \'%s\''%( dbsSiteQuery )
363  for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ):
364  if lineSite.find( dataset ) != -1:
365  foundSE = True
366  break
367  if not foundSE:
368  if debug:
369  print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
370  break
371  validVersion = version
372  if debug:
373  print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
374  if numberOfFiles == 0:
375  break
376  filePath = line.replace( '\n', '' )
377  if debug:
378  print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
379  fileCount += 1
380  if fileCount > skipFiles:
381  filePaths.append( filePath )
382  if not numberOfFiles < 0:
383  if numberOfFiles <= len( filePaths ):
384  break
385  if validVersion > 0:
386  if numberOfFiles == 0 and debug:
387  print '%s DEBUG: No files requested'%( self._label )
388  break
389 
390  # Check output and return
391  if validVersion == 0:
392  print '%s INFO : No RelVal file(s) found at all in datasets \'%s*\' on SE \'%s\''%( self._label, datasetAll, domainSE )
393  if debug:
394  self.messageEmptyList()
395  elif len( filePaths ) == 0:
396  print '%s INFO : No RelVal file(s) picked up in dataset \'%s\''%( self._label, dataset )
397  if debug:
398  self.messageEmptyList()
399  elif len( filePaths ) < numberOfFiles:
400  print '%s INFO : Only %i RelVal file(s) instead of %i picked up in dataset \'%s\''%( self._label, len( filePaths ), numberOfFiles, dataset )
401 
402  if debug:
403  print '%s DEBUG: returning %i file(s):\n%s'%( self._label, len( filePaths ), filePaths )
404  return filePaths
405 
406 pickRelValInputFiles = PickRelValInputFiles()
communicate
static void * communicate(void *obj)
Definition: DQMNet.cc:1246
cmsswVersionTools.PickRelValInputFiles
Automatic pick-up of RelVal input files
Definition: cmsswVersionTools.py:19
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5