CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Private Member Functions | Private Attributes | Static Private Attributes
dataset.Dataset Class Reference
Inheritance diagram for dataset.Dataset:
dataset.BaseDataset

Public Member Functions

def __init__
 
def __init__
 
def buildListOfBadFiles
 
def buildListOfFiles
 
def convertTimeToRun
 
def datasetSnippet
 
def dataType
 
def dump_cff
 
def extractFileSizes
 
def fileInfoList
 
def fileList
 
def forcerunrange
 
def getForceRunRangeFunction
 
def getPrimaryDatasetEntries
 
def magneticField
 
def magneticFieldForRun
 
def name
 
def parentDataset
 
def predefined
 
def printInfo
 
def runList
 
- Public Member Functions inherited from dataset.BaseDataset
def __init__
 def init(self, name, user, pattern='. More...
 
def buildListOfBadFiles
 
def buildListOfFiles
 
def extractFileSizes
 
def getPrimaryDatasetEntries
 
def listOfFiles
 
def listOfGoodFiles
 
def listOfGoodFilesWithPrescale
 
def printFiles
 
def printInfo
 

Public Attributes

 bad_files
 
 castorDir
 
 files
 
 filesAndSizes
 
 good_files
 
 lfnDir
 
 maskExists
 
 report
 
- Public Attributes inherited from dataset.BaseDataset
 bad_files
 
 dbsInstance
 MM. More...
 
 files
 
 filesAndSizes
 
 good_files
 
 name
 
 pattern
 
 primaryDatasetEntries
 MM. More...
 
 report
 
 run_range
 
 user
 

Private Member Functions

def __chunks
 
def __createSnippet
 
def __dateString
 
def __datetime
 
def __find_ge
 
def __find_lt
 
def __findInJson
 
def __getData
 
def __getDataType
 
def __getFileInfoList
 
def __getMagneticField
 
def __getMagneticFieldForRun
 
def __getParentDataset
 
def __getRunList
 

Private Attributes

 __alreadyStored
 
 __cmssw
 
 __cmsswrelease
 
 __dasLimit
 
 __dataType
 
 __fileInfoList
 
 __fileList
 
 __filename
 
 __firstusedrun
 
 __lastusedrun
 
 __magneticField
 
 __name
 
 __official
 
 __origName
 
 __parentDataset
 
 __parentFileInfoList
 
 __parentFileList
 
 __predefined
 
 __runList
 

Static Private Attributes

tuple __dummy_source_template
 

Detailed Description

Definition at line 14 of file dataset.py.

Constructor & Destructor Documentation

def dataset.Dataset.__init__ (   self,
  datasetName,
  dasLimit = 0,
  tryPredefinedFirst = True,
  cmssw = os.environ["CMSSW_BASE"],
  cmsswrelease = os.environ["CMSSW_RELEASE_BASE"] 
)

Definition at line 16 of file dataset.py.

Referenced by dataset.Dataset.__init__().

16 
17  cmssw = os.environ["CMSSW_BASE"], cmsswrelease = os.environ["CMSSW_RELEASE_BASE"]):
18  self.__name = datasetName
19  self.__origName = datasetName
20  self.__dasLimit = dasLimit
21  self.__fileList = None
22  self.__fileInfoList = None
23  self.__runList = None
24  self.__alreadyStored = False
25  self.__cmssw = cmssw
26  self.__cmsswrelease = cmsswrelease
27  self.__firstusedrun = None
28  self.__lastusedrun = None
29  self.__parentDataset = None
30  self.__parentFileList = None
31  self.__parentFileInfoList = None
32 
33  # check, if dataset name matches CMS dataset naming scheme
34  if re.match( r'/.+/.+/.+', self.__name ):
35  self.__official = True
36  fileName = "Dataset" + self.__name.replace("/","_") + "_cff.py"
37  else:
38  self.__official = False
39  fileName = self.__name + "_cff.py"
40 
41  searchPath1 = os.path.join( self.__cmssw, "python",
42  "Alignment", "OfflineValidation",
43  fileName )
44  searchPath2 = os.path.join( self.__cmssw, "src",
45  "Alignment", "OfflineValidation",
46  "python", fileName )
47  searchPath3 = os.path.join( self.__cmsswrelease,
48  "python", "Alignment",
49  "OfflineValidation", fileName )
50  if self.__official and not tryPredefinedFirst:
51  self.__predefined = False
52  elif os.path.exists( searchPath1 ):
53  self.__predefined = True
54  self.__filename = searchPath1
55  elif os.path.exists( searchPath2 ):
56  msg = ("The predefined dataset '%s' does exist in '%s', but "
57  "you need to run 'scram b' first."
58  %( self.__name, searchPath2 ))
59  if self.__official:
60  print msg
61  print "Getting the data from DAS again. To go faster next time, run scram b."
62  else:
63  raise AllInOneError( msg )
64  elif os.path.exists( searchPath3 ):
65  self.__predefined = True
66  self.__filename = searchPath3
67  elif self.__official:
68  self.__predefined = False
69  else:
70  msg = ("The predefined dataset '%s' does not exist. Please "
71  "create it first or check for typos."%( self.__name ))
72  raise AllInOneError( msg )
73 
74  if self.__predefined and self.__official:
75  self.__name = "Dataset" + self.__name.replace("/","_")
76 
77  self.__dataType = self.__getDataType()
78  self.__magneticField = self.__getMagneticField()
dataset.Dataset.__getMagneticField
def __getMagneticField
Definition: dataset.py:376
dataset.Dataset.__getDataType
def __getDataType
Definition: dataset.py:341
def dataset.Dataset.__init__ (   self,
  name,
  user,
  pattern = '.*root' 
)

Definition at line 265 of file dataset.py.

References dataset.Dataset.__init__().

265 
266  def __init__(self, name, user, pattern='.*root'):
267  self.lfnDir = castorBaseDir(user) + name
268  self.castorDir = castortools.lfnToCastor( self.lfnDir )
269  self.maskExists = False
270  self.report = None
271  super(Dataset, self).__init__(name, user, pattern)

Member Function Documentation

def dataset.Dataset.__chunks (   self,
  theList,
  n 
)
private 
Yield successive n-sized chunks from theList.

Definition at line 79 of file dataset.py.

Referenced by dataset.Dataset.__createSnippet().

79 
80  def __chunks( self, theList, n ):
81  """ Yield successive n-sized chunks from theList.
82  """
83  for i in xrange( 0, len( theList ), n ):
84  yield theList[i:i+n]
def dataset.Dataset.__createSnippet (   self,
  jsonPath = None,
  begin = None,
  end = None,
  firstRun = None,
  lastRun = None,
  repMap = None,
  crab = False,
  parent = False 
)
private

Definition at line 117 of file dataset.py.

References dataset.Dataset.__chunks(), dataset.Dataset.__dummy_source_template, dataset.Dataset.__findInJson(), dataset.Dataset.__firstusedrun, dataset.Dataset.__getRunList(), dataset.Dataset.__lastusedrun, dataset.Dataset.convertTimeToRun(), dataset.Dataset.fileList(), dataset.Dataset.getForceRunRangeFunction(), join(), list(), bookConverter.max, min(), dataset.Dataset.predefined(), and split.

Referenced by dataset.Dataset.datasetSnippet(), and dataset.Dataset.dump_cff().

117 
118  crab = False, parent = False ):
119  if firstRun:
120  firstRun = int( firstRun )
121  if lastRun:
122  lastRun = int( lastRun )
123  if ( begin and firstRun ) or ( end and lastRun ):
124  msg = ( "The Usage of "
125  + "'begin' & 'firstRun' " * int( bool( begin and
126  firstRun ) )
127  + "and " * int( bool( ( begin and firstRun ) and
128  ( end and lastRun ) ) )
129  + "'end' & 'lastRun' " * int( bool( end and lastRun ) )
130  + "is ambigous." )
131  raise AllInOneError( msg )
132  if begin or end:
133  ( firstRun, lastRun ) = self.convertTimeToRun(
134  begin = begin, end = end, firstRun = firstRun,
135  lastRun = lastRun )
136  if ( firstRun and lastRun ) and ( firstRun > lastRun ):
137  msg = ( "The lower time/runrange limit ('begin'/'firstRun') "
138  "chosen is greater than the upper time/runrange limit "
139  "('end'/'lastRun').")
140  raise AllInOneError( msg )
141  if self.predefined() and (jsonPath or begin or end or firstRun or lastRun):
142  msg = ( "The parameters 'JSON', 'begin', 'end', 'firstRun', and 'lastRun'"
143  "only work for official datasets, not predefined _cff.py files" )
144  raise AllInOneError( msg )
145  goodLumiSecStr = ""
146  lumiStr = ""
147  lumiSecExtend = ""
148  if firstRun or lastRun or jsonPath:
149  goodLumiSecStr = ( "lumiSecs = cms.untracked."
150  "VLuminosityBlockRange()\n" )
151  lumiStr = " lumisToProcess = lumiSecs,\n"
152  if not jsonPath:
153  selectedRunList = self.__getRunList()
154  if firstRun:
155  selectedRunList = [ run for run in selectedRunList \
156  if self.__findInJson(run, "run_number") >= firstRun ]
157  if lastRun:
158  selectedRunList = [ run for run in selectedRunList \
159  if self.__findInJson(run, "run_number") <= lastRun ]
160  lumiList = [ str( self.__findInJson(run, "run_number") ) + ":1-" \
161  + str( self.__findInJson(run, "run_number") ) + ":max" \
162  for run in selectedRunList ]
163  splitLumiList = list( self.__chunks( lumiList, 255 ) )
164  else:
165  theLumiList = None
166  try:
167  theLumiList = LumiList ( filename = jsonPath )
168  except ValueError:
169  pass
170 
171  if theLumiList is not None:
172  allRuns = theLumiList.getRuns()
173  runsToRemove = []
174  for run in allRuns:
175  if firstRun and int( run ) < firstRun:
176  runsToRemove.append( run )
177  if lastRun and int( run ) > lastRun:
178  runsToRemove.append( run )
179  theLumiList.removeRuns( runsToRemove )
180  splitLumiList = list( self.__chunks(
181  theLumiList.getCMSSWString().split(','), 255 ) )
182  else:
183  with open(jsonPath) as f:
184  jsoncontents = f.read()
185  if "process.source.lumisToProcess" in jsoncontents:
186  msg = "%s is not a json file, but it seems to be a CMSSW lumi selection cff snippet. Trying to use it" % jsonPath
187  if firstRun or lastRun:
188  msg += ("\n (after applying firstRun and/or lastRun)")
189  msg += ".\nPlease note that, depending on the format of this file, it may not work as expected."
190  msg += "\nCheck your config file to make sure that it worked properly."
191  print msg
192 
193  runlist = self.__getRunList()
194  if firstRun or lastRun:
195  self.__firstusedrun = -1
196  self.__lastusedrun = -1
197  jsoncontents = re.sub("\d+:(\d+|max)-\d+:(\d+|max)", self.getForceRunRangeFunction(firstRun, lastRun), jsoncontents)
198  self.__firstusedrun = max(self.__firstusedrun, int(self.__findInJson(runlist[0],"run_number")))
199  self.__lastusedrun = min(self.__lastusedrun, int(self.__findInJson(runlist[-1],"run_number")))
200  else:
201  self.__firstusedrun = int(self.__findInJson(runlist[0],"run_number"))
202  self.__lastusedrun = int(self.__findInJson(runlist[-1],"run_number"))
203  lumiSecExtend = jsoncontents
204  splitLumiList = [[""]]
205 
206  if not len(splitLumiList[0][0]) == 0:
207  lumiSecStr = [ "',\n'".join( lumis ) \
208  for lumis in splitLumiList ]
209  lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
210  for lumis in lumiSecStr ]
211  lumiSecExtend = "\n".join( lumiSecStr )
212  runlist = self.__getRunList()
213  self.__firstusedrun = max(int(splitLumiList[0][0].split(":")[0]), int(self.__findInJson(runlist[0],"run_number")))
214  self.__lastusedrun = min(int(splitLumiList[-1][-1].split(":")[0]), int(self.__findInJson(runlist[-1],"run_number")))
215  else:
216  runlist = self.__getRunList()
217  self.__firstusedrun = int(self.__findInJson(self.__getRunList()[0],"run_number"))
218  self.__lastusedrun = int(self.__findInJson(self.__getRunList()[-1],"run_number"))
219 
220  if crab:
221  files = ""
222  else:
223  splitFileList = list( self.__chunks( self.fileList(), 255 ) )
224  fileStr = [ "',\n'".join( files ) for files in splitFileList ]
225  fileStr = [ "readFiles.extend( [\n'" + files + "'\n] )" \
226  for files in fileStr ]
227  files = "\n".join( fileStr )
228 
229  if parent:
230  splitParentFileList = list( self.__chunks( self.fileList(parent = True), 255 ) )
231  parentFileStr = [ "',\n'".join( parentFiles ) for parentFiles in splitParentFileList ]
232  parentFileStr = [ "secFiles.extend( [\n'" + parentFiles + "'\n] )" \
233  for parentFiles in parentFileStr ]
234  parentFiles = "\n".join( parentFileStr )
235  files += "\n\n" + parentFiles
236 
237 
238  theMap = repMap
239  theMap["files"] = files
240  theMap["json"] = jsonPath
241  theMap["lumiStr"] = lumiStr
242  theMap["goodLumiSecStr"] = goodLumiSecStr%( theMap )
243  theMap["lumiSecExtend"] = lumiSecExtend
244  if crab:
245  dataset_snippet = self.__dummy_source_template%( theMap )
246  else:
247  dataset_snippet = self.__source_template%( theMap )
248  return dataset_snippet
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
min
T min(T a, T b)
Definition: MathUtil.h:58
dataset.Dataset.convertTimeToRun
def convertTimeToRun
Definition: dataset.py:603
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
dataset.Dataset.__dummy_source_template
tuple __dummy_source_template
Definition: dataset.py:103
dataset.Dataset.__getRunList
def __getRunList
Definition: dataset.py:576
dataset.Dataset.getForceRunRangeFunction
def getForceRunRangeFunction
Definition: dataset.py:306
split
double split
Definition: MVATrainer.cc:139
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def dataset.Dataset.__dateString (   self,
  date 
)
private

Definition at line 598 of file dataset.py.

References dataset.Dataset.convertTimeToRun().

Referenced by dataset.Dataset.convertTimeToRun().

598 
599  def __dateString(self, date):
600  return str(date.year) + str(date.month).zfill(2) + str(date.day).zfill(2)
dataset.Dataset.__dateString
def __dateString
Definition: dataset.py:598
def dataset.Dataset.__datetime (   self,
  stringForDas 
)
private

Definition at line 589 of file dataset.py.

Referenced by dataset.Dataset.convertTimeToRun().

589 
590  def __datetime(self, stringForDas):
591  if len(stringForDas) != 8:
592  raise AllInOneError(stringForDas + " is not a valid date string.\n"
593  + "DAS accepts dates in the form 'yyyymmdd'")
594  year = stringForDas[:4]
595  month = stringForDas[4:6]
596  day = stringForDas[6:8]
597  return datetime.date(int(year), int(month), int(day))
def dataset.Dataset.__find_ge (   self,
  a,
  x 
)
private

Definition at line 256 of file dataset.py.

Referenced by dataset.Dataset.convertTimeToRun().

256 
257  def __find_ge( self, a, x):
258  'Find leftmost item greater than or equal to x'
259  i = bisect.bisect_left( a, x )
260  if i != len( a ):
261  return i
262  raise ValueError
def dataset.Dataset.__find_lt (   self,
  a,
  x 
)
private

Definition at line 249 of file dataset.py.

Referenced by dataset.Dataset.convertTimeToRun().

249 
250  def __find_lt( self, a, x ):
251  'Find rightmost value less than x'
252  i = bisect.bisect_left( a, x )
253  if i:
254  return i-1
255  raise ValueError
def dataset.Dataset.__findInJson (   self,
  jsondict,
  strings 
)
private

Definition at line 263 of file dataset.py.

References dataset.Dataset.__findInJson().

Referenced by dataset.Dataset.__createSnippet(), dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.__getParentDataset(), dataset.Dataset.__getRunList(), dataset.Dataset.convertTimeToRun(), and dataset.Dataset.fileList().

263 
264  def __findInJson(self, jsondict, strings):
265  if isinstance(strings, str):
266  strings = [ strings ]
267 
268  if len(strings) == 0:
269  return jsondict
270  if isinstance(jsondict,dict):
271  if strings[0] in jsondict:
272  try:
273  return self.__findInJson(jsondict[strings[0]], strings[1:])
274  except KeyError:
275  pass
276  else:
277  for a in jsondict:
278  if strings[0] in a:
279  try:
280  return self.__findInJson(a[strings[0]], strings[1:])
281  except (TypeError, KeyError): #TypeError because a could be a string and contain strings[0]
282  pass
283  #if it's not found
284  raise KeyError("Can't find " + strings[0])
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
def dataset.Dataset.__getData (   self,
  dasQuery,
  dasLimit = 0 
)
private

Definition at line 311 of file dataset.py.

References dataset.Dataset.__findInJson().

Referenced by dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.__getParentDataset(), dataset.Dataset.__getRunList(), and dataset.Dataset.convertTimeToRun().

311 
312  def __getData( self, dasQuery, dasLimit = 0 ):
313  dasData = das_client.get_data( 'https://cmsweb.cern.ch',
314  dasQuery, 0, dasLimit, False )
315  if isinstance(dasData, str):
316  jsondict = json.loads( dasData )
317  else:
318  jsondict = dasData
319  # Check, if the DAS query fails
320  try:
321  error = self.__findInJson(jsondict,["data","error"])
322  except KeyError:
323  error = None
324  if error or self.__findInJson(jsondict,"status") != 'ok' or "data" not in jsondict:
325  jsonstr = str(jsondict)
326  if len(jsonstr) > 10000:
327  jsonfile = "das_query_output_%i.txt"
328  i = 0
329  while os.path.lexists(jsonfile % i):
330  i += 1
331  jsonfile = jsonfile % i
332  theFile = open( jsonfile, "w" )
333  theFile.write( jsonstr )
334  theFile.close()
335  msg = "The DAS query returned an error. The output is very long, and has been stored in:\n" + jsonfile
336  else:
337  msg = "The DAS query returned a error. Here is the output\n" + jsonstr
338  msg += "\nIt's possible that this was a server error. If so, it may work if you try again later"
339  raise AllInOneError(msg)
340  return self.__findInJson(jsondict,"data")
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
def dataset.Dataset.__getDataType (   self)
private

Definition at line 341 of file dataset.py.

References dataset.Dataset.__filename, dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__name, dataset.Dataset.__predefined, ElectronMVAID.ElectronMVAID.name, counter.Counter.name, entry.name, average.Average.name, histograms.Histograms.name, cond::persistency::TAG::NAME.name, TmModule.name, cond::persistency::GLOBAL_TAG::NAME.name, core.autovars.NTupleVariable.name, cond::persistency::TAG::TIME_TYPE.name, genericValidation.GenericValidation.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::COND_LOG_TABLE::EXECTIME.name, cond::persistency::TAG::OBJECT_TYPE.name, preexistingValidation.PreexistingValidation.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, ora::RecordSpecImpl::Item.name, cond::persistency::COND_LOG_TABLE::IOVTAG.name, cond::persistency::TAG::SYNCHRONIZATION.name, cond::persistency::GLOBAL_TAG::RELEASE.name, cond::persistency::COND_LOG_TABLE::USERTEXT.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, cond::persistency::TAG::END_OF_VALIDITY.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::TAG::DESCRIPTION.name, cond::persistency::GTEditorData.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, FWTGeoRecoGeometry::Info.name, Types._Untracked.name, cond::persistency::TAG::INSERTION_TIME.name, cond::persistency::TAG::MODIFICATION_TIME.name, dataset.BaseDataset.name, personalPlayback.Applet.name, ParameterSet.name, PixelDCSObject< class >::Item.name, analyzer.Analyzer.name, DQMRivetClient::LumiOption.name, MagCylinder.name, alignment.Alignment.name, ParSet.name, DQMRivetClient::ScaleFactorOption.name, SingleObjectCondition.name, EgHLTOfflineSummaryClient::SumHistBinData.name, XMLHTRZeroSuppressionLoader::_loaderBaseConfig.name, DQMGenericClient::EfficOption.name, XMLRBXPedestalsLoader::_loaderBaseConfig.name, cond::persistency::GTProxyData.name, core.autovars.NTupleObjectType.name, MyWatcher.name, edm::PathTimingSummary.name, cond::TimeTypeSpecs.name, lumi::TriggerInfo.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, edm::PathSummary.name, PixelEndcapLinkMaker::Item.name, perftools::EdmEventSize::BranchRecord.name, RecAnalyzerMinbias.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, FWTableViewManager::TableEntry.name, PixelBarrelLinkMaker::Item.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, Mapper::definition< ScannerT >.name, EcalLogicID.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, ExpressionHisto< T >.name, McSelector.name, RecoSelector.name, XMLProcessor::_loaderBaseConfig.name, DQMGenericClient::ProfileOption.name, TreeCrawler.Package.name, cond::persistency::PAYLOAD::HASH.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, cond::persistency::PAYLOAD::DATA.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, cond::persistency::PAYLOAD::VERSION.name, MagGeoBuilderFromDDD::volumeHandle.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, DQMGenericClient::NormOption.name, options.ConnectionHLTMenu.name, DQMGenericClient::CDOption.name, cond::TagInfo_t.name, h4DSegm.name, PhysicsTools::Calibration::Variable.name, looper.Looper.name, EDMtoMEConverter.name, MEtoEDM< T >::MEtoEDMObject.name, cond::persistency::IOV::TAG_NAME.name, cond::persistency::IOV::SINCE.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, MuonGeometrySanityCheckPoint.name, config.Analyzer.name, config.Service.name, core.autovars.NTupleSubObject.name, core.autovars.NTupleObject.name, h2DSegm.name, options.HLTProcessOptions.name, DQMNet::WaitObject.name, AlpgenParameterName.name, SiStripMonitorDigi.name, core.autovars.NTupleCollection.name, cond::persistency::TAG_MIGRATION::SOURCE_ACCOUNT.name, cond::persistency::TAG_MIGRATION::SOURCE_TAG.name, cond::persistency::TAG_MIGRATION::TAG_NAME.name, cond::persistency::TAG_MIGRATION::STATUS_CODE.name, cond::persistency::TAG_MIGRATION::INSERTION_TIME.name, FastTimerService::LuminosityDescription.name, cond::persistency::PAYLOAD_MIGRATION::SOURCE_ACCOUNT.name, cond::persistency::PAYLOAD_MIGRATION::SOURCE_TOKEN.name, cond::persistency::PAYLOAD_MIGRATION::PAYLOAD_HASH.name, cond::persistency::PAYLOAD_MIGRATION::INSERTION_TIME.name, conddblib.Tag.name, conddblib.GlobalTag.name, personalPlayback.FrameworkJob.name, plotscripts.SawTeethFunction.name, FastTimerService::ProcessDescription.name, hTMaxCell.name, cscdqm::ParHistoDef.name, BeautifulSoup.Tag.name, TiXmlAttribute.name, BeautifulSoup.SoupStrainer.name, and python.rootplot.root2matplotlib.replace().

Referenced by dataset.Dataset.dataType().

341 
342  def __getDataType( self ):
343  if self.__predefined:
344  with open(self.__filename) as f:
345  datatype = None
346  for line in f.readlines():
347  if line.startswith("#data type: "):
348  if datatype is not None:
349  raise AllInOneError(self.__filename + " has multiple 'data type' lines.")
350  datatype = line.replace("#data type: ", "").replace("\n","")
351  return datatype
352  return "unknown"
353 
354  dasQuery_type = ( 'dataset dataset=%s | grep dataset.datatype,'
355  'dataset.name'%( self.__name ) )
356  data = self.__getData( dasQuery_type )
357 
358  try:
359  return self.__findInJson(data, ["dataset", "datatype"])
360  except KeyError:
361  print ("Cannot find the datatype of the dataset '%s'\n"
362  "It may not be possible to automatically find the magnetic field,\n"
363  "and you will not be able run in CRAB mode"
364  %( self.name() ))
365  return "unknown"
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
dataset.Dataset.__getDataType
def __getDataType
Definition: dataset.py:341
def dataset.Dataset.__getFileInfoList (   self,
  dasLimit,
  parent = False 
)
private

Definition at line 510 of file dataset.py.

References dataset.Dataset.__fileInfoList, dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__name, dataset.Dataset.__parentFileInfoList, dataset.Dataset.__predefined, ElectronMVAID.ElectronMVAID.name, counter.Counter.name, entry.name, average.Average.name, histograms.Histograms.name, cond::persistency::TAG::NAME.name, TmModule.name, cond::persistency::GLOBAL_TAG::NAME.name, core.autovars.NTupleVariable.name, cond::persistency::TAG::TIME_TYPE.name, genericValidation.GenericValidation.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::COND_LOG_TABLE::EXECTIME.name, cond::persistency::TAG::OBJECT_TYPE.name, preexistingValidation.PreexistingValidation.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, ora::RecordSpecImpl::Item.name, cond::persistency::COND_LOG_TABLE::IOVTAG.name, cond::persistency::TAG::SYNCHRONIZATION.name, cond::persistency::GLOBAL_TAG::RELEASE.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, cond::persistency::COND_LOG_TABLE::USERTEXT.name, cond::persistency::TAG::END_OF_VALIDITY.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::TAG::DESCRIPTION.name, cond::persistency::GTEditorData.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, FWTGeoRecoGeometry::Info.name, Types._Untracked.name, cond::persistency::TAG::INSERTION_TIME.name, dataset.BaseDataset.name, cond::persistency::TAG::MODIFICATION_TIME.name, personalPlayback.Applet.name, ParameterSet.name, PixelDCSObject< class >::Item.name, analyzer.Analyzer.name, DQMRivetClient::LumiOption.name, MagCylinder.name, alignment.Alignment.name, ParSet.name, DQMRivetClient::ScaleFactorOption.name, SingleObjectCondition.name, EgHLTOfflineSummaryClient::SumHistBinData.name, XMLHTRZeroSuppressionLoader::_loaderBaseConfig.name, DQMGenericClient::EfficOption.name, XMLRBXPedestalsLoader::_loaderBaseConfig.name, cond::persistency::GTProxyData.name, core.autovars.NTupleObjectType.name, MyWatcher.name, edm::PathTimingSummary.name, lumi::TriggerInfo.name, cond::TimeTypeSpecs.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, edm::PathSummary.name, PixelEndcapLinkMaker::Item.name, perftools::EdmEventSize::BranchRecord.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, RecAnalyzerMinbias.name, FWTableViewManager::TableEntry.name, PixelBarrelLinkMaker::Item.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, Mapper::definition< ScannerT >.name, EcalLogicID.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, ExpressionHisto< T >.name, McSelector.name, RecoSelector.name, cond::persistency::PAYLOAD::HASH.name, DQMGenericClient::ProfileOption.name, TreeCrawler.Package.name, XMLProcessor::_loaderBaseConfig.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, cond::persistency::PAYLOAD::DATA.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, cond::persistency::PAYLOAD::VERSION.name, MagGeoBuilderFromDDD::volumeHandle.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, DQMGenericClient::NormOption.name, options.ConnectionHLTMenu.name, DQMGenericClient::CDOption.name, cond::TagInfo_t.name, h4DSegm.name, PhysicsTools::Calibration::Variable.name, looper.Looper.name, EDMtoMEConverter.name, MEtoEDM< T >::MEtoEDMObject.name, cond::persistency::IOV::TAG_NAME.name, cond::persistency::IOV::SINCE.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, MuonGeometrySanityCheckPoint.name, config.Analyzer.name, config.Service.name, core.autovars.NTupleSubObject.name, core.autovars.NTupleObject.name, h2DSegm.name, options.HLTProcessOptions.name, DQMNet::WaitObject.name, AlpgenParameterName.name, SiStripMonitorDigi.name, core.autovars.NTupleCollection.name, cond::persistency::TAG_MIGRATION::SOURCE_ACCOUNT.name, cond::persistency::TAG_MIGRATION::SOURCE_TAG.name, cond::persistency::TAG_MIGRATION::TAG_NAME.name, cond::persistency::TAG_MIGRATION::STATUS_CODE.name, cond::persistency::TAG_MIGRATION::INSERTION_TIME.name, FastTimerService::LuminosityDescription.name, cond::persistency::PAYLOAD_MIGRATION::SOURCE_ACCOUNT.name, cond::persistency::PAYLOAD_MIGRATION::SOURCE_TOKEN.name, cond::persistency::PAYLOAD_MIGRATION::PAYLOAD_HASH.name, cond::persistency::PAYLOAD_MIGRATION::INSERTION_TIME.name, conddblib.Tag.name, conddblib.GlobalTag.name, personalPlayback.FrameworkJob.name, plotscripts.SawTeethFunction.name, FastTimerService::ProcessDescription.name, hTMaxCell.name, cscdqm::ParHistoDef.name, BeautifulSoup.Tag.name, TiXmlAttribute.name, BeautifulSoup.SoupStrainer.name, and dataset.Dataset.parentDataset().

Referenced by dataset.Dataset.fileInfoList().

510 
511  def __getFileInfoList( self, dasLimit, parent = False ):
512  if self.__predefined:
513  if parent:
514  extendstring = "secFiles.extend"
515  else:
516  extendstring = "readFiles.extend"
517  with open(self.__fileName) as f:
518  files = []
519  copy = False
520  for line in f.readlines():
521  if "]" in line:
522  copy = False
523  if copy:
524  files.append({name: line.translate(None, "', " + '"')})
525  if extendstring in line and "[" in line and "]" not in line:
526  copy = True
527  return files
528 
529  if self.__fileInfoList and not parent:
530  return self.__fileInfoList
531  if self.__parentFileInfoList and parent:
532  return self.__parentFileInfoList
533 
534  if parent:
535  searchdataset = self.parentDataset()
536  else:
537  searchdataset = self.__name
538  dasQuery_files = ( 'file dataset=%s | grep file.name, file.nevents, '
539  'file.creation_time, '
540  'file.modification_time'%( searchdataset ) )
541  print "Requesting file information for '%s' from DAS..."%( searchdataset ),
542  data = self.__getData( dasQuery_files, dasLimit )
543  print "Done."
544  data = [ self.__findInJson(entry,"file") for entry in data ]
545  if len( data ) == 0:
546  msg = ("No files are available for the dataset '%s'. This can be "
547  "due to a typo or due to a DAS problem. Please check the "
548  "spelling of the dataset and/or retry to run "
549  "'validateAlignments.py'."%( self.name() ))
550  raise AllInOneError( msg )
551  fileInformationList = []
552  for file in data:
553  fileName = 'unknown'
554  try:
555  fileName = self.__findInJson(file, "name")
556  fileCreationTime = self.__findInJson(file, "creation_time")
557  fileNEvents = self.__findInJson(file, "nevents")
558  except KeyError:
559  print ("DAS query gives bad output for file '%s'. Skipping it.\n"
560  "It may work if you try again later.") % fileName
561  fileNEvents = 0
562  # select only non-empty files
563  if fileNEvents == 0:
564  continue
565  fileDict = { "name": fileName,
566  "creation_time": fileCreationTime,
567  "nevents": fileNEvents
568  }
569  fileInformationList.append( fileDict )
570  fileInformationList.sort( key=lambda info: self.__findInJson(info,"name") )
571  if parent:
572  self.__parentFileInfoList = fileInformationList
573  else:
574  self.__fileInfoList = fileInformationList
575  return fileInformationList
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
dataset.Dataset.__getFileInfoList
def __getFileInfoList
Definition: dataset.py:510
dataset.Dataset.parentDataset
def parentDataset
Definition: dataset.py:681
def dataset.Dataset.__getMagneticField (   self)
private

Definition at line 376 of file dataset.py.

References dataset.Dataset.__cmssw, dataset.Dataset.__cmsswrelease, dataset.Dataset.__dataType, dataset.Dataset.__filename, dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__name, dataset.Dataset.__predefined, and python.rootplot.root2matplotlib.replace().

Referenced by dataset.Dataset.magneticField().

376 
377  def __getMagneticField( self ):
378  Bfieldlocation = os.path.join( self.__cmssw, "python", "Configuration", "StandardSequences" )
379  if not os.path.isdir(Bfieldlocation):
380  Bfieldlocation = os.path.join( self.__cmsswrelease, "python", "Configuration", "StandardSequences" )
381  Bfieldlist = [ f.replace("_cff.py",'') \
382  for f in os.listdir(Bfieldlocation) \
383  if f.startswith("MagneticField_") and f.endswith("_cff.py") ]
384  Bfieldlist.sort( key = lambda Bfield: -len(Bfield) ) #Put it in order of decreasing length, so that searching in the name gives the longer match
385 
386  if self.__predefined:
387  with open(self.__filename) as f:
388  datatype = None
389  Bfield = None
390  for line in f.readlines():
391  if line.startswith("#data type: "):
392  if datatype is not None:
393  raise AllInOneError(self.__filename + " has multiple 'data type' lines.")
394  datatype = line.replace("#data type: ", "").replace("\n","")
395  datatype = datatype.split("#")[0].strip()
396  if line.startswith("#magnetic field: "):
397  if Bfield is not None:
398  raise AllInOneError(self.__filename + " has multiple 'magnetic field' lines.")
399  Bfield = line.replace("#magnetic field: ", "").replace("\n","")
400  Bfield = Bfield.split("#")[0].strip()
401  if Bfield is not None:
402  Bfield = Bfield.split(",")[0]
403  if Bfield in Bfieldlist or Bfield == "unknown":
404  return Bfield
405  #===========================================================================
406  #For compatibility with already written datasets - remove this at some point
407  #(until the next === line)
408  #It's currently June 2015, anytime starting in 2016 is more than safe
409  elif Bfield == "AutoFromDBCurrent":
410  return "MagneticField"
411  elif "MagneticField_" + Bfield in Bfieldlist:
412  return "MagneticField_" + Bfield
413  #===========================================================================
414  else:
415  print "Your dataset has magnetic field '%s', which does not exist in your CMSSW version!" % Bfield
416  print "Using Bfield='unknown' - this will revert to the default"
417  return "unknown"
418  elif datatype == "data":
419  return "MagneticField" #this should be in the "#magnetic field" line, but for safety in case it got messed up
420  else:
421  return "unknown"
422 
423  if self.__dataType == "data":
424  return "MagneticField"
425 
426  dasQuery_B = ( 'dataset dataset=%s'%( self.__name ) ) #try to find the magnetic field from DAS
427  data = self.__getData( dasQuery_B ) #it seems to be there for the newer (7X) MC samples, except cosmics
428 
429  try:
430  Bfield = self.__findInJson(data, ["dataset", "mcm", "sequences", "magField"])
431  if Bfield in Bfieldlist:
432  return Bfield
433  elif Bfield == "38T" or Bfield == "38T_PostLS1":
434  return "MagneticField"
435  elif "MagneticField_" + Bfield in Bfieldlist:
436  return "MagneticField_" + Bfield
437  elif Bfield == "":
438  pass
439  else:
440  print "Your dataset has magnetic field '%s', which does not exist in your CMSSW version!" % Bfield
441  print "Using Bfield='unknown' - this will revert to the default magnetic field"
442  return "unknown"
443  except KeyError:
444  pass
445 
446  for possibleB in Bfieldlist:
447  if (possibleB != "MagneticField"
448  and possibleB.replace("MagneticField_","") in self.__name.replace("TkAlCosmics0T", "")):
449  #final attempt - try to identify the dataset from the name
450  #all cosmics dataset names contain "TkAlCosmics0T"
451  if possibleB == "MagneticField_38T" or possibleB == "MagneticField_38T_PostLS1":
452  return "MagneticField"
453  return possibleB
454 
455  return "unknown"
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
dataset.Dataset.__getMagneticField
def __getMagneticField
Definition: dataset.py:376
def dataset.Dataset.__getMagneticFieldForRun (   self,
  run = -1,
  tolerance = 0.5 
)
private 
For MC, this returns the same as the previous function.
   For data, it gets the magnetic field from the runs.  This is important for
   deciding which template to use for offlinevalidation

Definition at line 456 of file dataset.py.

References dataset.Dataset.__dataType, dataset.Dataset.__filename, dataset.Dataset.__findInJson(), dataset.Dataset.__firstusedrun, dataset.Dataset.__getData(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.__lastusedrun, dataset.Dataset.__magneticField, dataset.Dataset.__name, dataset.Dataset.__predefined, funct.abs(), python.rootplot.root2matplotlib.replace(), and split.

Referenced by dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.dump_cff(), and dataset.Dataset.magneticFieldForRun().

456 
457  def __getMagneticFieldForRun( self, run = -1, tolerance = 0.5 ):
458  """For MC, this returns the same as the previous function.
459  For data, it gets the magnetic field from the runs. This is important for
460  deciding which template to use for offlinevalidation
461  """
462  if self.__dataType == "mc" and self.__magneticField == "MagneticField":
463  return 3.8 #For 3.8T MC the default MagneticField is used
464  if "T" in self.__magneticField:
465  Bfield = self.__magneticField.split("T")[0].replace("MagneticField_","")
466  try:
467  return float(Bfield) / 10.0 #e.g. 38T and 38T_PostLS1 both return 3.8
468  except ValueError:
469  pass
470  if self.__predefined:
471  with open(self.__filename) as f:
472  Bfield = None
473  for line in f.readlines():
474  if line.startswith("#magnetic field: ") and "," in line:
475  if Bfield is not None:
476  raise AllInOneError(self.__filename + " has multiple 'magnetic field' lines.")
477  return float(line.replace("#magnetic field: ", "").split(",")[1].split("#")[0].strip())
478 
479  if run > 0:
480  dasQuery = ('run = %s'%run) #for data
481  data = self.__getData(dasQuery)
482  try:
483  return self.__findInJson(data, ["run","bfield"])
484  except KeyError:
485  return "unknown Can't get the magnetic field for run %s from DAS" % run
486 
487  #run < 0 - find B field for the first and last runs, and make sure they're compatible
488  # (to within tolerance)
489  #NOT FOOLPROOF! The magnetic field might go up and then down, or vice versa
490  if self.__firstusedrun is None or self.__lastusedrun is None:
491  return "unknown Can't get the exact magnetic field for the dataset until data has been retrieved from DAS."
492  firstrunB = self.__getMagneticFieldForRun(self.__firstusedrun)
493  lastrunB = self.__getMagneticFieldForRun(self.__lastusedrun)
494  try:
495  if abs(firstrunB - lastrunB) <= tolerance:
496  return .5*(firstrunB + lastrunB)
497  print firstrunB, lastrunB, tolerance
498  return ("unknown The beginning and end of your run range for %s\n"
499  "have different magnetic fields (%s, %s)!\n"
500  "Try limiting the run range using firstRun, lastRun, begin, end, or JSON,\n"
501  "or increasing the tolerance (in dataset.py) from %s.") % (self.__name, firstrunB, lastrunB, tolerance)
502  except TypeError:
503  try:
504  if "unknown" in firstrunB:
505  return firstrunB
506  else:
507  return lastrunB
508  except TypeError:
509  return lastrunB
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
dataset.Dataset.__getMagneticFieldForRun
def __getMagneticFieldForRun
Definition: dataset.py:456
funct::abs
Abs< T >::type abs(const T &t)
Definition: Abs.h:22
split
double split
Definition: MVATrainer.cc:139
def dataset.Dataset.__getParentDataset (   self)
private

Definition at line 366 of file dataset.py.

References dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), and dataset.Dataset.__name.

Referenced by dataset.Dataset.parentDataset().

366 
367  def __getParentDataset( self ):
368  dasQuery = "parent dataset=" + self.__name
369  data = self.__getData( dasQuery )
370  try:
371  return self.__findInJson(data, ["parent", "name"])
372  except KeyError:
373  raise AllInOneError("Cannot find the parent of the dataset '" + self.__name + "'\n"
374  "Here is the DAS output:\n" + str(jsondict) +
375  "\nIt's possible that this was a server error. If so, it may work if you try again later")
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
dataset.Dataset.__getParentDataset
def __getParentDataset
Definition: dataset.py:366
def dataset.Dataset.__getRunList (   self)
private

Definition at line 576 of file dataset.py.

References dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__name, and dataset.Dataset.__runList.

Referenced by dataset.Dataset.__createSnippet(), dataset.Dataset.convertTimeToRun(), and dataset.Dataset.runList().

576 
577  def __getRunList( self ):
578  if self.__runList:
579  return self.__runList
580  dasQuery_runs = ( 'run dataset=%s | grep run.run_number,'
581  'run.creation_time'%( self.__name ) )
582  print "Requesting run information for '%s' from DAS..."%( self.__name ),
583  data = self.__getData( dasQuery_runs )
584  print "Done."
585  data = [ self.__findInJson(entry,"run") for entry in data ]
586  data.sort( key = lambda run: self.__findInJson(run, "run_number") )
587  self.__runList = data
588  return data
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
dataset.Dataset.__getRunList
def __getRunList
Definition: dataset.py:576
def dataset.Dataset.buildListOfBadFiles (   self) 
fills the list of bad files from the IntegrityCheck log.

When the integrity check file is not available,
files are considered as good.

Definition at line 276 of file dataset.py.

276 
277  def buildListOfBadFiles(self):
278  '''fills the list of bad files from the IntegrityCheck log.
279 
280  When the integrity check file is not available,
281  files are considered as good.'''
282  mask = "IntegrityCheck"
283 
284  self.bad_files = {}
285  self.good_files = []
286 
287  file_mask = castortools.matchingFiles(self.castorDir, '^%s_.*\.txt$' % mask)
288  if file_mask:
289  from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
290  p = PublishToFileSystem(mask)
291  report = p.get(self.castorDir)
292  if report is not None and report:
293  self.maskExists = True
294  self.report = report
295  dup = report.get('ValidDuplicates',{})
296  for name, status in report['Files'].iteritems():
297  # print name, status
298  if not status[0]:
299  self.bad_files[name] = 'MarkedBad'
300  elif dup.has_key(name):
301  self.bad_files[name] = 'ValidDup'
302  else:
303  self.good_files.append( name )
304  else:
305  raise IntegrityCheckError( "ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found" )
dataset.Dataset.buildListOfBadFiles
def buildListOfBadFiles
Definition: dataset.py:276
def dataset.Dataset.buildListOfFiles (   self,
  pattern = '.*root' 
) 
fills list of files, taking all root files matching the pattern in the castor dir

Definition at line 272 of file dataset.py.

272 
273  def buildListOfFiles(self, pattern='.*root'):
274  '''fills list of files, taking all root files matching the pattern in the castor dir'''
275  self.files = castortools.matchingFiles( self.castorDir, pattern )
dataset.Dataset.buildListOfFiles
def buildListOfFiles
Definition: dataset.py:272
def dataset.Dataset.convertTimeToRun (   self,
  begin = None,
  end = None,
  firstRun = None,
  lastRun = None,
  shortTuple = True 
)

Definition at line 603 of file dataset.py.

References dataset.Dataset.__dateString(), dataset.Dataset.__datetime(), dataset.Dataset.__find_ge(), dataset.Dataset.__find_lt(), dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__getRunList(), and dataset.Dataset.__name.

Referenced by dataset.Dataset.__createSnippet(), and dataset.Dataset.__dateString().

603 
604  shortTuple = True ):
605  if ( begin and firstRun ) or ( end and lastRun ):
606  msg = ( "The Usage of "
607  + "'begin' & 'firstRun' " * int( bool( begin and
608  firstRun ) )
609  + "and " * int( bool( ( begin and firstRun ) and
610  ( end and lastRun ) ) )
611  + "'end' & 'lastRun' " * int( bool( end and lastRun ) )
612  + "is ambigous." )
613  raise AllInOneError( msg )
614 
615  if begin or end:
616  runList = [ self.__findInJson(run, "run_number") for run in self.__getRunList() ]
617 
618  if begin:
619  lastdate = begin
620  for delta in [ 1, 5, 10, 20, 30 ]: #try searching for about 2 months after begin
621  firstdate = lastdate
622  lastdate = self.__dateString(self.__datetime(firstdate) + datetime.timedelta(delta))
623  dasQuery_begin = "run date between[%s,%s]" % (firstdate, lastdate)
624  begindata = self.__getData(dasQuery_begin)
625  if len(begindata) > 0:
626  begindata.sort(key = lambda run: self.__findInJson(run, ["run", "run_number"]))
627  try:
628  runIndex = self.__find_ge( runList, self.__findInJson(begindata[0], ["run", "run_number"]))
629  except ValueError:
630  msg = ( "Your 'begin' is after the creation time of the last "
631  "run in the dataset\n'%s'"%( self.__name ) )
632  raise AllInOneError( msg )
633  firstRun = runList[runIndex]
634  begin = None
635  break
636 
637  if begin:
638  raise AllInOneError("No runs within a reasonable time interval after your 'begin'."
639  "Try using a 'begin' that has runs soon after it (within 2 months at most)")
640 
641  if end:
642  firstdate = end
643  for delta in [ 1, 5, 10, 20, 30 ]: #try searching for about 2 months before end
644  lastdate = firstdate
645  firstdate = self.__dateString(self.__datetime(lastdate) - datetime.timedelta(delta))
646  dasQuery_end = "run date between[%s,%s]" % (firstdate, lastdate)
647  enddata = self.__getData(dasQuery_end)
648  if len(enddata) > 0:
649  enddata.sort(key = lambda run: self.__findInJson(run, ["run", "run_number"]))
650  try:
651  runIndex = self.__find_lt( runList, self.__findInJson(enddata[-1], ["run", "run_number"]))
652  except ValueError:
653  msg = ( "Your 'end' is before the creation time of the first "
654  "run in the dataset\n'%s'"%( self.__name ) )
655  raise AllInOneError( msg )
656  lastRun = runList[runIndex]
657  end = None
658  break
659 
660  if end:
661  raise AllInOneError("No runs within a reasonable time interval before your 'end'."
662  "Try using an 'end' that has runs soon before it (within 2 months at most)")
663 
664  if shortTuple:
665  return firstRun, lastRun
666  else:
667  return begin, end, firstRun, lastRun
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
dataset.Dataset.__getRunList
def __getRunList
Definition: dataset.py:576
dataset.Dataset.__dateString
def __dateString
Definition: dataset.py:598
def dataset.Dataset.datasetSnippet (   self,
  jsonPath = None,
  begin = None,
  end = None,
  firstRun = None,
  lastRun = None,
  crab = False,
  parent = False 
)

Definition at line 687 of file dataset.py.

References dataset.Dataset.__createSnippet(), dataset.Dataset.__filename, dataset.Dataset.__name, dataset.Dataset.__official, dataset.Dataset.__origName, dataset.Dataset.__predefined, and dataset.Dataset.dump_cff().

Referenced by dataset.Dataset.parentDataset().

687 
688  firstRun = None, lastRun = None, crab = False, parent = False ):
689  if self.__predefined and parent:
690  with open(self.__filename) as f:
691  if "secFiles.extend" not in f.read():
692  msg = ("The predefined dataset '%s' does not contain secondary files, "
693  "which your validation requires!") % self.__name
694  if self.__official:
695  self.__name = self.__origName
696  self.__predefined = False
697  print msg
698  print ("Retreiving the files from DAS. You will be asked if you want "
699  "to overwrite the old dataset.\n"
700  "It will still be compatible with validations that don't need secondary files.")
701  else:
702  raise AllInOneError(msg)
703 
704  if self.__predefined:
705  snippet = ("process.load(\"Alignment.OfflineValidation.%s_cff\")\n"
706  "process.maxEvents = cms.untracked.PSet(\n"
707  " input = cms.untracked.int32(.oO[nEvents]Oo. / .oO[parallelJobs]Oo.)\n"
708  ")\n"
709  "process.source.skipEvents=cms.untracked.uint32(.oO[nIndex]Oo.*.oO[nEvents]Oo./.oO[parallelJobs]Oo.)"
710  %(self.__name))
711  if not parent:
712  with open(self.__filename) as f:
713  if "secFiles.extend" in f.read():
714  snippet += "\nprocess.source.secondaryFileNames = cms.untracked.vstring()"
715  return snippet
716  theMap = { "process": "process.",
717  "tab": " " * len( "process." ),
718  "nEvents": ".oO[nEvents]Oo. / .oO[parallelJobs]Oo.",
719  "skipEventsString": "process.source.skipEvents=cms.untracked.uint32(.oO[nIndex]Oo.*.oO[nEvents]Oo./.oO[parallelJobs]Oo.)\n",
720  "importCms": "",
721  "header": ""
722  }
723  datasetSnippet = self.__createSnippet( jsonPath = jsonPath,
724  begin = begin,
725  end = end,
726  firstRun = firstRun,
727  lastRun = lastRun,
728  repMap = theMap,
729  crab = crab,
730  parent = parent )
731  if jsonPath == "" and begin == "" and end == "" and firstRun == "" and lastRun == "":
732  try:
733  self.dump_cff(parent = parent)
734  except AllInOneError, e:
735  print "Can't store the dataset as a cff:"
736  print e
737  print "This may be inconvenient in the future, but will not cause a problem for this validation."
738  return datasetSnippet
dataset.Dataset.__createSnippet
def __createSnippet
Definition: dataset.py:117
def dataset.Dataset.dataType (   self)

Definition at line 668 of file dataset.py.

References dataset.Dataset.__dataType, and dataset.Dataset.__getDataType().

668 
669  def dataType( self ):
670  if not self.__dataType:
671  self.__dataType = self.__getDataType()
672  return self.__dataType
dataset.Dataset.__getDataType
def __getDataType
Definition: dataset.py:341
def dataset.Dataset.dump_cff (   self,
  outName = None,
  jsonPath = None,
  begin = None,
  end = None,
  firstRun = None,
  lastRun = None,
  parent = False 
)

Definition at line 740 of file dataset.py.

References dataset.Dataset.__alreadyStored, dataset.Dataset.__cmssw, dataset.Dataset.__createSnippet(), dataset.Dataset.__dataType, dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.__magneticField, dataset.Dataset.__name, python.rootplot.root2matplotlib.replace(), and split.

Referenced by dataset.Dataset.datasetSnippet().

740 
741  end = None, firstRun = None, lastRun = None, parent = False ):
742  if self.__alreadyStored:
743  return
744  self.__alreadyStored = True
745  if outName == None:
746  outName = "Dataset" + self.__name.replace("/", "_")
747  packageName = os.path.join( "Alignment", "OfflineValidation" )
748  if not os.path.exists( os.path.join(
749  self.__cmssw, "src", packageName ) ):
750  msg = ("You try to store the predefined dataset'%s'.\n"
751  "For that you need to check out the package '%s' to your "
752  "private relase area in\n"%( outName, packageName )
753  + self.__cmssw )
754  raise AllInOneError( msg )
755  theMap = { "process": "",
756  "tab": "",
757  "nEvents": str( -1 ),
758  "skipEventsString": "",
759  "importCms": "import FWCore.ParameterSet.Config as cms\n",
760  "header": "#Do not delete or (unless you know what you're doing) change these comments\n"
761  "#%(name)s\n"
762  "#data type: %(dataType)s\n"
763  "#magnetic field: .oO[magneticField]Oo.\n" #put in magnetic field later
764  %{"name": self.__name, #need to create the snippet before getting the magnetic field
765  "dataType": self.__dataType} #so that we know the first and last runs
766  }
767  dataset_cff = self.__createSnippet( jsonPath = jsonPath,
768  begin = begin,
769  end = end,
770  firstRun = firstRun,
771  lastRun = lastRun,
772  repMap = theMap,
773  parent = parent)
774  magneticField = self.__magneticField
775  if magneticField == "MagneticField":
776  magneticField = "%s, %s #%s" % (magneticField,
777  str(self.__getMagneticFieldForRun()).replace("\n"," ").split("#")[0].strip(),
778  "Use MagneticField_cff.py; the number is for determining which track selection to use."
779  )
780  dataset_cff = dataset_cff.replace(".oO[magneticField]Oo.",magneticField)
781  filePath = os.path.join( self.__cmssw, "src", packageName,
782  "python", outName + "_cff.py" )
783  if os.path.exists( filePath ):
784  existMsg = "The predefined dataset '%s' already exists.\n"%( outName )
785  askString = "Do you want to overwrite it? [y/n]\n"
786  inputQuery = existMsg + askString
787  while True:
788  userInput = raw_input( inputQuery ).lower()
789  if userInput == "y":
790  break
791  elif userInput == "n":
792  return
793  else:
794  inputQuery = askString
795  print ( "The predefined dataset '%s' will be stored in the file\n"
796  %( outName )
797  + filePath +
798  "\nFor future use you have to do 'scram b'." )
799  print
800  theFile = open( filePath, "w" )
801  theFile.write( dataset_cff )
802  theFile.close()
803  return
dataset.Dataset.__getMagneticFieldForRun
def __getMagneticFieldForRun
Definition: dataset.py:456
dataset.Dataset.__createSnippet
def __createSnippet
Definition: dataset.py:117
split
double split
Definition: MVATrainer.cc:139
def dataset.Dataset.extractFileSizes (   self) 
Get the file size for each file, from the eos ls -l command.

Definition at line 306 of file dataset.py.

References dataset.EOSDataset.castorDir, and dataset.Dataset.castorDir.

306 
307  def extractFileSizes(self):
308  '''Get the file size for each file, from the eos ls -l command.'''
309  # EOS command does not work in tier3
310  lsout = castortools.runXRDCommand(self.castorDir,'dirlist')[0]
311  lsout = lsout.split('\n')
312  self.filesAndSizes = {}
313  for entry in lsout:
314  values = entry.split()
315  if( len(values) != 5):
316  continue
317  # using full abs path as a key.
318  file = '/'.join([self.lfnDir, values[4].split("/")[-1]])
319  size = values[1]
320  self.filesAndSizes[file] = size
dataset.Dataset.extractFileSizes
def extractFileSizes
Definition: dataset.py:306
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
if
if(conf.exists("allCellsPositionCalc"))
Definition: Basic2DGenericPFlowClusterizer.cc:58
split
double split
Definition: MVATrainer.cc:139
def dataset.Dataset.fileInfoList (   self,
  parent = False 
)

Definition at line 819 of file dataset.py.

References dataset.Dataset.__dasLimit, and dataset.Dataset.__getFileInfoList().

Referenced by dataset.Dataset.fileList().

819 
820  def fileInfoList( self, parent = False ):
821  return self.__getFileInfoList( self.__dasLimit, parent )
dataset.Dataset.__getFileInfoList
def __getFileInfoList
Definition: dataset.py:510
dataset.Dataset.fileInfoList
def fileInfoList
Definition: dataset.py:819
def dataset.Dataset.fileList (   self,
  parent = False 
)

Definition at line 804 of file dataset.py.

References dataset.Dataset.__fileList, dataset.Dataset.__findInJson(), dataset.Dataset.__parentFileList, and dataset.Dataset.fileInfoList().

Referenced by dataset.Dataset.__createSnippet().

804 
805  def fileList( self, parent = False ):
806  if self.__fileList and not parent:
807  return self.__fileList
808  if self.__parentFileList and parent:
809  return self.__parentFileList
810 
811  fileList = [ self.__findInJson(fileInfo,"name") \
812  for fileInfo in self.fileInfoList(parent) ]
813 
814  if not parent:
815  self.__fileList = fileList
816  else:
817  self.__parentFileList = fileList
818  return fileList
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:263
dataset.Dataset.fileInfoList
def fileInfoList
Definition: dataset.py:819
def dataset.Dataset.forcerunrange (   self,
  firstRun,
  lastRun,
  s 
) 
s must be in the format run1:lum1-run2:lum2

Definition at line 285 of file dataset.py.

References dataset.Dataset.__firstusedrun, dataset.Dataset.__lastusedrun, and split.

Referenced by dataset.Dataset.getForceRunRangeFunction().

285 
286  def forcerunrange(self, firstRun, lastRun, s):
287  """s must be in the format run1:lum1-run2:lum2"""
288  s = s.group()
289  run1 = s.split("-")[0].split(":")[0]
290  lum1 = s.split("-")[0].split(":")[1]
291  run2 = s.split("-")[1].split(":")[0]
292  lum2 = s.split("-")[1].split(":")[1]
293  if int(run2) < firstRun or int(run1) > lastRun:
294  return ""
295  if int(run1) < firstRun or firstRun < 0:
296  run1 = firstRun
297  lum1 = 1
298  if int(run2) > lastRun:
299  run2 = lastRun
300  lum2 = "max"
301  if int(run1) < self.__firstusedrun or self.__firstusedrun < 0:
302  self.__firstusedrun = int(run1)
303  if int(run2) > self.__lastusedrun:
304  self.__lastusedrun = int(run2)
305  return "%s:%s-%s:%s" % (run1, lum1, run2, lum2)
dataset.Dataset.forcerunrange
def forcerunrange
Definition: dataset.py:285
split
double split
Definition: MVATrainer.cc:139
def dataset.Dataset.getForceRunRangeFunction (   self,
  firstRun,
  lastRun 
)

Definition at line 306 of file dataset.py.

References dataset.Dataset.forcerunrange().

Referenced by dataset.Dataset.__createSnippet().

306 
307  def getForceRunRangeFunction(self, firstRun, lastRun):
308  def forcerunrangefunction(s):
309  return self.forcerunrange(firstRun, lastRun, s)
310  return forcerunrangefunction
dataset.Dataset.forcerunrange
def forcerunrange
Definition: dataset.py:285
dataset.Dataset.getForceRunRangeFunction
def getForceRunRangeFunction
Definition: dataset.py:306
def dataset.Dataset.getPrimaryDatasetEntries (   self)

Definition at line 326 of file dataset.py.

References runall.testit.report, dataset.BaseDataset.report, ALIUtils.report, and WorkFlowRunner.WorkFlowRunner.report.

326 
327  def getPrimaryDatasetEntries(self):
328  if self.report is not None and self.report:
329  return int(self.report.get('PrimaryDatasetEntries',-1))
330  return -1
331 
dataset.Dataset.getPrimaryDatasetEntries
def getPrimaryDatasetEntries
Definition: dataset.py:326
def dataset.Dataset.magneticField (   self)

Definition at line 673 of file dataset.py.

References dataset.Dataset.__getMagneticField(), and dataset.Dataset.__magneticField.

673 
674  def magneticField( self ):
675  if not self.__magneticField:
676  self.__magneticField = self.__getMagneticField()
677  return self.__magneticField
dataset.Dataset.magneticField
def magneticField
Definition: dataset.py:673
dataset.Dataset.__getMagneticField
def __getMagneticField
Definition: dataset.py:376
def dataset.Dataset.magneticFieldForRun (   self,
  run = -1 
)

Definition at line 678 of file dataset.py.

References dataset.Dataset.__getMagneticFieldForRun().

678 
679  def magneticFieldForRun( self, run = -1 ):
680  return self.__getMagneticFieldForRun(run)
dataset.Dataset.__getMagneticFieldForRun
def __getMagneticFieldForRun
Definition: dataset.py:456
dataset.Dataset.magneticFieldForRun
def magneticFieldForRun
Definition: dataset.py:678
def dataset.Dataset.name (   self)

Definition at line 822 of file dataset.py.

References dataset.Dataset.__name.

Referenced by cuy.divideElement.__init__(), cuy.plotElement.__init__(), cuy.additionElement.__init__(), cuy.superimposeElement.__init__(), cuy.graphElement.__init__(), config.CFG.__str__(), VIDSelectorBase.VIDSelectorBase.initialize(), and Vispa.Views.PropertyView.Property.valueChanged().

822 
823  def name( self ):
824  return self.__name
def dataset.Dataset.parentDataset (   self)

Definition at line 681 of file dataset.py.

References dataset.Dataset.__getParentDataset(), dataset.Dataset.__parentDataset, and dataset.Dataset.datasetSnippet().

Referenced by dataset.Dataset.__getFileInfoList().

681 
682  def parentDataset( self ):
683  if not self.__parentDataset:
684  self.__parentDataset = self.__getParentDataset()
685  return self.__parentDataset
dataset.Dataset.parentDataset
def parentDataset
Definition: dataset.py:681
dataset.Dataset.__getParentDataset
def __getParentDataset
Definition: dataset.py:366
def dataset.Dataset.predefined (   self)

Definition at line 825 of file dataset.py.

References dataset.Dataset.__predefined.

Referenced by dataset.Dataset.__createSnippet().

825 
826  def predefined( self ):
827  return self.__predefined
def dataset.Dataset.printInfo (   self)

Definition at line 321 of file dataset.py.

References dataset.EOSDataset.castorDir, dataset.Dataset.castorDir, dataset.Dataset.lfnDir, ElectronMVAID.ElectronMVAID.name, counter.Counter.name, average.Average.name, entry.name, histograms.Histograms.name, TmModule.name, cond::persistency::GLOBAL_TAG::NAME.name, core.autovars.NTupleVariable.name, cond::persistency::TAG::NAME.name, genericValidation.GenericValidation.name, cond::persistency::TAG::TIME_TYPE.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::TAG::OBJECT_TYPE.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, preexistingValidation.PreexistingValidation.name, cond::persistency::COND_LOG_TABLE::EXECTIME.name, cond::persistency::TAG::SYNCHRONIZATION.name, cond::persistency::GLOBAL_TAG::RELEASE.name, ora::RecordSpecImpl::Item.name, cond::persistency::COND_LOG_TABLE::IOVTAG.name, cond::persistency::COND_LOG_TABLE::USERTEXT.name, cond::persistency::TAG::END_OF_VALIDITY.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, cond::persistency::GTEditorData.name, cond::persistency::TAG::DESCRIPTION.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, FWTGeoRecoGeometry::Info.name, Types._Untracked.name, cond::persistency::TAG::INSERTION_TIME.name, cond::persistency::TAG::MODIFICATION_TIME.name, dataset.BaseDataset.name, personalPlayback.Applet.name, ParameterSet.name, PixelDCSObject< class >::Item.name, analyzer.Analyzer.name, DQMRivetClient::LumiOption.name, MagCylinder.name, alignment.Alignment.name, ParSet.name, DQMRivetClient::ScaleFactorOption.name, SingleObjectCondition.name, EgHLTOfflineSummaryClient::SumHistBinData.name, DQMGenericClient::EfficOption.name, cond::persistency::GTProxyData.name, core.autovars.NTupleObjectType.name, XMLRBXPedestalsLoader::_loaderBaseConfig.name, XMLHTRZeroSuppressionLoader::_loaderBaseConfig.name, MyWatcher.name, edm::PathTimingSummary.name, cond::TimeTypeSpecs.name, lumi::TriggerInfo.name, edm::PathSummary.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, perftools::EdmEventSize::BranchRecord.name, PixelEndcapLinkMaker::Item.name, FWTableViewManager::TableEntry.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, RecAnalyzerMinbias.name, PixelBarrelLinkMaker::Item.name, Mapper::definition< ScannerT >.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, EcalLogicID.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, ExpressionHisto< T >.name, McSelector.name, RecoSelector.name, DQMGenericClient::ProfileOption.name, cond::persistency::PAYLOAD::HASH.name, TreeCrawler.Package.name, XMLProcessor::_loaderBaseConfig.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, cond::persistency::PAYLOAD::DATA.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, MagGeoBuilderFromDDD::volumeHandle.name, cond::persistency::PAYLOAD::VERSION.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, DQMGenericClient::NormOption.name, options.ConnectionHLTMenu.name, DQMGenericClient::CDOption.name, h4DSegm.name, cond::TagInfo_t.name, PhysicsTools::Calibration::Variable.name, looper.Looper.name, EDMtoMEConverter.name, MEtoEDM< T >::MEtoEDMObject.name, cond::persistency::IOV::TAG_NAME.name, cond::persistency::IOV::SINCE.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, MuonGeometrySanityCheckPoint.name, config.Analyzer.name, config.Service.name, core.autovars.NTupleSubObject.name, core.autovars.NTupleObject.name, h2DSegm.name, options.HLTProcessOptions.name, DQMNet::WaitObject.name, AlpgenParameterName.name, SiStripMonitorDigi.name, core.autovars.NTupleCollection.name, cond::persistency::TAG_MIGRATION::SOURCE_ACCOUNT.name, cond::persistency::TAG_MIGRATION::SOURCE_TAG.name, cond::persistency::TAG_MIGRATION::TAG_NAME.name, cond::persistency::TAG_MIGRATION::STATUS_CODE.name, cond::persistency::TAG_MIGRATION::INSERTION_TIME.name, FastTimerService::LuminosityDescription.name, cond::persistency::PAYLOAD_MIGRATION::SOURCE_ACCOUNT.name, cond::persistency::PAYLOAD_MIGRATION::SOURCE_TOKEN.name, cond::persistency::PAYLOAD_MIGRATION::PAYLOAD_HASH.name, cond::persistency::PAYLOAD_MIGRATION::INSERTION_TIME.name, conddblib.Tag.name, conddblib.GlobalTag.name, personalPlayback.FrameworkJob.name, plotscripts.SawTeethFunction.name, FastTimerService::ProcessDescription.name, hTMaxCell.name, cscdqm::ParHistoDef.name, BeautifulSoup.Tag.name, TiXmlAttribute.name, and BeautifulSoup.SoupStrainer.name.

321 
322  def printInfo(self):
323  print 'sample : ' + self.name
324  print 'LFN : ' + self.lfnDir
325  print 'Castor path : ' + self.castorDir
def dataset.Dataset.runList (   self)

Definition at line 828 of file dataset.py.

References dataset.Dataset.__getRunList(), and dataset.Dataset.__runList.

828 
829  def runList( self ):
830  if self.__runList:
831  return self.__runList
832  return self.__getRunList()
833 
dataset.Dataset.__getRunList
def __getRunList
Definition: dataset.py:576

Member Data Documentation

dataset.Dataset.__alreadyStored
private

Definition at line 23 of file dataset.py.

Referenced by dataset.Dataset.dump_cff().

dataset.Dataset.__cmssw
private

Definition at line 24 of file dataset.py.

Referenced by dataset.Dataset.__getMagneticField(), and dataset.Dataset.dump_cff().

dataset.Dataset.__cmsswrelease
private

Definition at line 25 of file dataset.py.

Referenced by dataset.Dataset.__getMagneticField().

dataset.Dataset.__dasLimit
private

Definition at line 19 of file dataset.py.

Referenced by dataset.Dataset.fileInfoList().

dataset.Dataset.__dataType
private

Definition at line 76 of file dataset.py.

Referenced by dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.dataType(), and dataset.Dataset.dump_cff().

tuple dataset.Dataset.__dummy_source_template
staticprivate
Initial value:
1 = ("readFiles = cms.untracked.vstring()\n"
2  "secFiles = cms.untracked.vstring()\n"
3  "%(process)ssource = cms.Source(\"PoolSource\",\n"
4  "%(tab)s secondaryFileNames ="
5  "secFiles,\n"
6  "%(tab)s fileNames = readFiles\n"
7  ")\n"
8  "readFiles.extend(['dummy_File.root'])\n"
9  "%(process)smaxEvents = cms.untracked.PSet( "
10  "input = cms.untracked.int32(%(nEvents)s) )\n"
11  "%(skipEventsString)s\n")

Definition at line 103 of file dataset.py.

Referenced by dataset.Dataset.__createSnippet().

dataset.Dataset.__fileInfoList
private

Definition at line 21 of file dataset.py.

Referenced by dataset.Dataset.__getFileInfoList().

dataset.Dataset.__fileList
private

Definition at line 20 of file dataset.py.

Referenced by dataset.Dataset.fileList().

dataset.Dataset.__filename
private

Definition at line 53 of file dataset.py.

Referenced by dataset.Dataset.__getDataType(), dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.datasetSnippet(), csvReporter.csvReporter.writeRow(), and csvReporter.csvReporter.writeRows().

dataset.Dataset.__firstusedrun
private

Definition at line 26 of file dataset.py.

Referenced by dataset.Dataset.__createSnippet(), dataset.Dataset.__getMagneticFieldForRun(), and dataset.Dataset.forcerunrange().

dataset.Dataset.__lastusedrun
private

Definition at line 27 of file dataset.py.

Referenced by dataset.Dataset.__createSnippet(), dataset.Dataset.__getMagneticFieldForRun(), and dataset.Dataset.forcerunrange().

dataset.Dataset.__magneticField
private

Definition at line 77 of file dataset.py.

Referenced by dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.dump_cff(), and dataset.Dataset.magneticField().

dataset.Dataset.__name
private

Definition at line 17 of file dataset.py.

Referenced by dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.__getParentDataset(), dataset.Dataset.__getRunList(), dataset.Dataset.convertTimeToRun(), dataset.Dataset.datasetSnippet(), dataset.Dataset.dump_cff(), Config.Process.dumpConfig(), Config.Process.dumpPython(), dataset.Dataset.name(), and Config.Process.name_().

dataset.Dataset.__official
private

Definition at line 34 of file dataset.py.

Referenced by dataset.Dataset.datasetSnippet().

dataset.Dataset.__origName
private

Definition at line 18 of file dataset.py.

Referenced by dataset.Dataset.datasetSnippet().

dataset.Dataset.__parentDataset
private

Definition at line 28 of file dataset.py.

Referenced by dataset.Dataset.parentDataset().

dataset.Dataset.__parentFileInfoList
private

Definition at line 30 of file dataset.py.

Referenced by dataset.Dataset.__getFileInfoList().

dataset.Dataset.__parentFileList
private

Definition at line 29 of file dataset.py.

Referenced by dataset.Dataset.fileList().

dataset.Dataset.__predefined
private

Definition at line 50 of file dataset.py.

Referenced by dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.datasetSnippet(), and dataset.Dataset.predefined().

dataset.Dataset.__runList
private

Definition at line 22 of file dataset.py.

Referenced by dataset.Dataset.__getRunList(), and dataset.Dataset.runList().

dataset.Dataset.bad_files

Definition at line 283 of file dataset.py.

dataset.Dataset.castorDir

Definition at line 267 of file dataset.py.

Referenced by dataset.Dataset.extractFileSizes(), and dataset.Dataset.printInfo().

dataset.Dataset.files

Definition at line 274 of file dataset.py.

dataset.Dataset.filesAndSizes

Definition at line 311 of file dataset.py.

dataset.Dataset.good_files

Definition at line 284 of file dataset.py.

dataset.Dataset.lfnDir

Definition at line 266 of file dataset.py.

Referenced by dataset.Dataset.printInfo().

dataset.Dataset.maskExists

Definition at line 268 of file dataset.py.

dataset.Dataset.report

Definition at line 269 of file dataset.py.

Referenced by addOnTests.testit.run().

Generated for CMSSW Reference Manual by doxygen 1.8.5