test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes | Private Member Functions | Private Attributes | Static Private Attributes
dataset.Dataset Class Reference
Inheritance diagram for dataset.Dataset:
dataset.BaseDataset

Public Member Functions

def __init__
 
def __init__
 
def buildListOfBadFiles
 
def buildListOfFiles
 
def convertTimeToRun
 
def datasetSnippet
 
def dataType
 
def dump_cff
 
def extractFileSizes
 
def fileInfoList
 
def fileList
 
def forcerunrange
 
def getForceRunRangeFunction
 
def getPrimaryDatasetEntries
 
def magneticField
 
def magneticFieldForRun
 
def name
 
def parentDataset
 
def predefined
 
def printInfo
 
def runList
 
- Public Member Functions inherited from dataset.BaseDataset
def __init__
 def init(self, name, user, pattern='. More...
 
def buildListOfBadFiles
 
def buildListOfFiles
 
def extractFileSizes
 
def getPrimaryDatasetEntries
 
def listOfFiles
 
def listOfGoodFiles
 
def listOfGoodFilesWithPrescale
 
def printFiles
 
def printInfo
 

Public Attributes

 bad_files
 
 castorDir
 
 files
 
 filesAndSizes
 
 good_files
 
 lfnDir
 
 maskExists
 
 report
 
- Public Attributes inherited from dataset.BaseDataset
 bad_files
 
 dbsInstance
 MM. More...
 
 files
 
 filesAndSizes
 
 good_files
 
 name
 
 pattern
 
 primaryDatasetEntries
 MM. More...
 
 report
 
 run_range
 
 user
 

Static Public Attributes

tuple dasData = das_client.get_data(dasQuery, dasLimit)
 
tuple error = self.__findInJson(jsondict,["data","error"])
 
 error = None
 
int i = 0
 
tuple jsondict = json.loads( dasData )
 
 jsondict = dasData
 
string jsonfile = "das_query_output_%i.txt"
 
 jsonfile = jsonfile%i
 
tuple jsonstr = self.__findInJson(jsondict,"reason")
 
string msg = "The DAS query returned an error. The output is very long, and has been stored in:\n"
 
tuple theFile = open( jsonfile, "w" )
 

Private Member Functions

def __chunks
 
def __createSnippet
 
def __dateString
 
def __datetime
 
def __find_ge
 
def __find_lt
 
def __findInJson
 
def __getData
 
def __getDataType
 
def __getFileInfoList
 
def __getMagneticField
 
def __getMagneticFieldForRun
 
def __getParentDataset
 
def __getRunList
 

Private Attributes

 __alreadyStored
 
 __cmssw
 
 __cmsswrelease
 
 __dasLimit
 
 __dataType
 
 __fileInfoList
 
 __fileList
 
 __filename
 
 __firstusedrun
 
 __lastusedrun
 
 __magneticField
 
 __name
 
 __official
 
 __origName
 
 __parentDataset
 
 __parentFileInfoList
 
 __parentFileList
 
 __predefined
 
 __runList
 

Static Private Attributes

tuple __dummy_source_template
 

Detailed Description

Definition at line 14 of file dataset.py.

Constructor & Destructor Documentation

def dataset.Dataset.__init__ (   self,
  datasetName,
  dasLimit = 0,
  tryPredefinedFirst = True,
  cmssw = os.environ["CMSSW_BASE"],
  cmsswrelease = os.environ["CMSSW_RELEASE_BASE"] 
)

Definition at line 16 of file dataset.py.

Referenced by dataset.Dataset.__init__().

16 
17  cmssw = os.environ["CMSSW_BASE"], cmsswrelease = os.environ["CMSSW_RELEASE_BASE"]):
18  self.__name = datasetName
19  self.__origName = datasetName
20  self.__dasLimit = dasLimit
21  self.__fileList = None
22  self.__fileInfoList = None
23  self.__runList = None
24  self.__alreadyStored = False
25  self.__cmssw = cmssw
26  self.__cmsswrelease = cmsswrelease
27  self.__firstusedrun = None
28  self.__lastusedrun = None
29  self.__parentDataset = None
30  self.__parentFileList = None
31  self.__parentFileInfoList = None
32 
33  # check, if dataset name matches CMS dataset naming scheme
34  if re.match( r'/.+/.+/.+', self.__name ):
35  self.__official = True
36  fileName = "Dataset" + self.__name.replace("/","_") + "_cff.py"
37  else:
38  self.__official = False
39  fileName = self.__name + "_cff.py"
40 
41  searchPath1 = os.path.join( self.__cmssw, "python",
42  "Alignment", "OfflineValidation",
43  fileName )
44  searchPath2 = os.path.join( self.__cmssw, "src",
45  "Alignment", "OfflineValidation",
46  "python", fileName )
47  searchPath3 = os.path.join( self.__cmsswrelease,
48  "python", "Alignment",
49  "OfflineValidation", fileName )
50  if self.__official and not tryPredefinedFirst:
51  self.__predefined = False
52  elif os.path.exists( searchPath1 ):
53  self.__predefined = True
54  self.__filename = searchPath1
55  elif os.path.exists( searchPath2 ):
56  msg = ("The predefined dataset '%s' does exist in '%s', but "
57  "you need to run 'scram b' first."
58  %( self.__name, searchPath2 ))
59  if self.__official:
60  print msg
61  print "Getting the data from DAS again. To go faster next time, run scram b."
62  else:
63  raise AllInOneError( msg )
64  elif os.path.exists( searchPath3 ):
65  self.__predefined = True
66  self.__filename = searchPath3
67  elif self.__official:
68  self.__predefined = False
69  else:
70  msg = ("The predefined dataset '%s' does not exist. Please "
71  "create it first or check for typos."%( self.__name ))
72  raise AllInOneError( msg )
73 
74  if self.__predefined and self.__official:
75  self.__name = "Dataset" + self.__name.replace("/","_")
76 
77  self.__dataType = self.__getDataType()
78  self.__magneticField = self.__getMagneticField()
dataset.Dataset.__getMagneticField
def __getMagneticField
Definition: dataset.py:406
dataset.Dataset.__getDataType
def __getDataType
Definition: dataset.py:371
def dataset.Dataset.__init__ (   self,
  name,
  user,
  pattern = '.*root' 
)

Definition at line 264 of file dataset.py.

References dataset.Dataset.__init__().

264 
265  def __init__(self, name, user, pattern='.*root'):
266  self.lfnDir = castorBaseDir(user) + name
267  self.castorDir = castortools.lfnToCastor( self.lfnDir )
268  self.maskExists = False
269  self.report = None
270  super(Dataset, self).__init__(name, user, pattern)

Member Function Documentation

def dataset.Dataset.__chunks (   self,
  theList,
  n 
)
private 
Yield successive n-sized chunks from theList.

Definition at line 79 of file dataset.py.

Referenced by dataset.Dataset.__createSnippet().

79 
80  def __chunks( self, theList, n ):
81  """ Yield successive n-sized chunks from theList.
82  """
83  for i in xrange( 0, len( theList ), n ):
84  yield theList[i:i+n]
def dataset.Dataset.__createSnippet (   self,
  jsonPath = None,
  begin = None,
  end = None,
  firstRun = None,
  lastRun = None,
  repMap = None,
  crab = False,
  parent = False 
)
private

Definition at line 117 of file dataset.py.

References dataset.Dataset.__chunks(), dataset.Dataset.__dummy_source_template, dataset.Dataset.__findInJson(), dataset.Dataset.__firstusedrun, dataset.Dataset.__getRunList(), dataset.Dataset.__lastusedrun, dataset.Dataset.convertTimeToRun(), dataset.Dataset.fileList(), dataset.Dataset.getForceRunRangeFunction(), join(), list(), bookConverter.max, min(), dataset.Dataset.predefined(), python.rootplot.root2matplotlib.replace(), and split.

Referenced by dataset.Dataset.datasetSnippet(), and dataset.Dataset.dump_cff().

117 
118  crab = False, parent = False ):
119  if firstRun:
120  firstRun = int( firstRun )
121  if lastRun:
122  lastRun = int( lastRun )
123  if ( begin and firstRun ) or ( end and lastRun ):
124  msg = ( "The Usage of "
125  + "'begin' & 'firstRun' " * int( bool( begin and
126  firstRun ) )
127  + "and " * int( bool( ( begin and firstRun ) and
128  ( end and lastRun ) ) )
129  + "'end' & 'lastRun' " * int( bool( end and lastRun ) )
130  + "is ambigous." )
131  raise AllInOneError( msg )
132  if begin or end:
133  ( firstRun, lastRun ) = self.convertTimeToRun(
134  begin = begin, end = end, firstRun = firstRun,
135  lastRun = lastRun )
136  if ( firstRun and lastRun ) and ( firstRun > lastRun ):
137  msg = ( "The lower time/runrange limit ('begin'/'firstRun') "
138  "chosen is greater than the upper time/runrange limit "
139  "('end'/'lastRun').")
140  raise AllInOneError( msg )
141  if self.predefined() and (jsonPath or begin or end or firstRun or lastRun):
142  msg = ( "The parameters 'JSON', 'begin', 'end', 'firstRun', and 'lastRun'"
143  "only work for official datasets, not predefined _cff.py files" )
144  raise AllInOneError( msg )
145  goodLumiSecStr = ""
146  lumiStr = ""
147  lumiSecExtend = ""
148  if firstRun or lastRun or jsonPath:
149  goodLumiSecStr = ( "lumiSecs = cms.untracked."
150  "VLuminosityBlockRange()\n" )
151  lumiStr = " lumisToProcess = lumiSecs,\n"
152  if not jsonPath:
153  selectedRunList = self.__getRunList()
154  if firstRun:
155  selectedRunList = [ run for run in selectedRunList \
156  if self.__findInJson(run, "run_number") >= firstRun ]
157  if lastRun:
158  selectedRunList = [ run for run in selectedRunList \
159  if self.__findInJson(run, "run_number") <= lastRun ]
160  lumiList = [ str( self.__findInJson(run, "run_number") ) + ":1-" \
161  + str( self.__findInJson(run, "run_number") ) + ":max" \
162  for run in selectedRunList ]
163  splitLumiList = list( self.__chunks( lumiList, 255 ) )
164  else:
165  theLumiList = None
166  try:
167  theLumiList = LumiList ( filename = jsonPath )
168  except ValueError:
169  pass
170 
171  if theLumiList is not None:
172  allRuns = theLumiList.getRuns()
173  runsToRemove = []
174  for run in allRuns:
175  if firstRun and int( run ) < firstRun:
176  runsToRemove.append( run )
177  if lastRun and int( run ) > lastRun:
178  runsToRemove.append( run )
179  theLumiList.removeRuns( runsToRemove )
180  splitLumiList = list( self.__chunks(
181  theLumiList.getCMSSWString().split(','), 255 ) )
182  if not (splitLumiList and splitLumiList[0] and splitLumiList[0][0]):
183  splitLumiList = None
184  else:
185  with open(jsonPath) as f:
186  jsoncontents = f.read()
187  if "process.source.lumisToProcess" in jsoncontents:
188  msg = "%s is not a json file, but it seems to be a CMSSW lumi selection cff snippet. Trying to use it" % jsonPath
189  if firstRun or lastRun:
190  msg += ("\n (after applying firstRun and/or lastRun)")
191  msg += ".\nPlease note that, depending on the format of this file, it may not work as expected."
192  msg += "\nCheck your config file to make sure that it worked properly."
193  print msg
194 
195  runlist = self.__getRunList()
196  if firstRun or lastRun:
197  self.__firstusedrun = -1
198  self.__lastusedrun = -1
199  jsoncontents = re.sub(r"\d+:(\d+|max)(-\d+:(\d+|max))?", self.getForceRunRangeFunction(firstRun, lastRun), jsoncontents)
200  jsoncontents = (jsoncontents.replace("'',\n","").replace("''\n","")
201  .replace('"",\n','').replace('""\n',''))
202  self.__firstusedrun = max(self.__firstusedrun, int(self.__findInJson(runlist[0],"run_number")))
203  self.__lastusedrun = min(self.__lastusedrun, int(self.__findInJson(runlist[-1],"run_number")))
204  if self.__lastusedrun < self.__firstusedrun:
205  jsoncontents = None
206  else:
207  self.__firstusedrun = int(self.__findInJson(runlist[0],"run_number"))
208  self.__lastusedrun = int(self.__findInJson(runlist[-1],"run_number"))
209  lumiSecExtend = jsoncontents
210  splitLumiList = None
211  else:
212  raise AllInOneError("%s is not a valid json file!" % jsonPath)
213 
214  if splitLumiList and splitLumiList[0] and splitLumiList[0][0]:
215  lumiSecStr = [ "',\n'".join( lumis ) \
216  for lumis in splitLumiList ]
217  lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \
218  for lumis in lumiSecStr ]
219  lumiSecExtend = "\n".join( lumiSecStr )
220  runlist = self.__getRunList()
221  self.__firstusedrun = max(int(splitLumiList[0][0].split(":")[0]), int(self.__findInJson(runlist[0],"run_number")))
222  self.__lastusedrun = min(int(splitLumiList[-1][-1].split(":")[0]), int(self.__findInJson(runlist[-1],"run_number")))
223  elif lumiSecExtend:
224  pass
225  else:
226  msg = "You are trying to run a validation without any runs! Check that:"
227  if firstRun or lastRun:
228  msg += "\n - firstRun and lastRun are correct for this dataset, and there are runs in between containing data"
229  if jsonPath:
230  msg += "\n - your JSON file is correct for this dataset, and the runs contain data"
231  if (firstRun or lastRun) and jsonPath:
232  msg += "\n - firstRun and lastRun are consistent with your JSON file"
233  if begin:
234  msg = msg.replace("firstRun", "begin")
235  if end:
236  msg = msg.replace("lastRun", "end")
237  raise AllInOneError(msg)
238 
239  else:
240  runlist = self.__getRunList()
241  self.__firstusedrun = int(self.__findInJson(self.__getRunList()[0],"run_number"))
242  self.__lastusedrun = int(self.__findInJson(self.__getRunList()[-1],"run_number"))
243 
244  if crab:
245  files = ""
246  else:
247  splitFileList = list( self.__chunks( self.fileList(), 255 ) )
248  fileStr = [ "',\n'".join( files ) for files in splitFileList ]
249  fileStr = [ "readFiles.extend( [\n'" + files + "'\n] )" \
250  for files in fileStr ]
251  files = "\n".join( fileStr )
252 
253  if parent:
254  splitParentFileList = list( self.__chunks( self.fileList(parent = True), 255 ) )
255  parentFileStr = [ "',\n'".join( parentFiles ) for parentFiles in splitParentFileList ]
256  parentFileStr = [ "secFiles.extend( [\n'" + parentFiles + "'\n] )" \
257  for parentFiles in parentFileStr ]
258  parentFiles = "\n".join( parentFileStr )
259  files += "\n\n" + parentFiles
260 
261 
262  theMap = repMap
263  theMap["files"] = files
264  theMap["json"] = jsonPath
265  theMap["lumiStr"] = lumiStr
266  theMap["goodLumiSecStr"] = goodLumiSecStr%( theMap )
267  theMap["lumiSecExtend"] = lumiSecExtend
268  if crab:
269  dataset_snippet = self.__dummy_source_template%( theMap )
270  else:
271  dataset_snippet = self.__source_template%( theMap )
272  return dataset_snippet
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:287
min
T min(T a, T b)
Definition: MathUtil.h:58
dataset.Dataset.convertTimeToRun
def convertTimeToRun
Definition: dataset.py:624
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
dataset.Dataset.__dummy_source_template
tuple __dummy_source_template
Definition: dataset.py:103
dataset.Dataset.__getRunList
def __getRunList
Definition: dataset.py:597
dataset.Dataset.getForceRunRangeFunction
def getForceRunRangeFunction
Definition: dataset.py:334
split
double split
Definition: MVATrainer.cc:139
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def dataset.Dataset.__dateString (   self,
  date 
)
private

Definition at line 619 of file dataset.py.

References dataset.Dataset.convertTimeToRun().

Referenced by dataset.Dataset.convertTimeToRun().

619 
620  def __dateString(self, date):
621  return str(date.year) + str(date.month).zfill(2) + str(date.day).zfill(2)
dataset.Dataset.__dateString
def __dateString
Definition: dataset.py:619
def dataset.Dataset.__datetime (   self,
  stringForDas 
)
private

Definition at line 610 of file dataset.py.

Referenced by dataset.Dataset.convertTimeToRun().

610 
611  def __datetime(self, stringForDas):
612  if len(stringForDas) != 8:
613  raise AllInOneError(stringForDas + " is not a valid date string.\n"
614  + "DAS accepts dates in the form 'yyyymmdd'")
615  year = stringForDas[:4]
616  month = stringForDas[4:6]
617  day = stringForDas[6:8]
618  return datetime.date(int(year), int(month), int(day))
def dataset.Dataset.__find_ge (   self,
  a,
  x 
)
private

Definition at line 280 of file dataset.py.

Referenced by dataset.Dataset.convertTimeToRun().

280 
281  def __find_ge( self, a, x):
282  'Find leftmost item greater than or equal to x'
283  i = bisect.bisect_left( a, x )
284  if i != len( a ):
285  return i
286  raise ValueError
def dataset.Dataset.__find_lt (   self,
  a,
  x 
)
private

Definition at line 273 of file dataset.py.

Referenced by dataset.Dataset.convertTimeToRun().

273 
274  def __find_lt( self, a, x ):
275  'Find rightmost value less than x'
276  i = bisect.bisect_left( a, x )
277  if i:
278  return i-1
279  raise ValueError
def dataset.Dataset.__findInJson (   self,
  jsondict,
  strings 
)
private

Definition at line 287 of file dataset.py.

References dataset.Dataset.__findInJson().

Referenced by dataset.Dataset.__createSnippet(), dataset.Dataset.__findInJson(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.__getParentDataset(), dataset.Dataset.__getRunList(), dataset.Dataset.convertTimeToRun(), and dataset.Dataset.fileList().

287 
288  def __findInJson(self, jsondict, strings):
289  if isinstance(strings, str):
290  strings = [ strings ]
291 
292  if len(strings) == 0:
293  return jsondict
294  if isinstance(jsondict,dict):
295  if strings[0] in jsondict:
296  try:
297  return self.__findInJson(jsondict[strings[0]], strings[1:])
298  except KeyError:
299  pass
300  else:
301  for a in jsondict:
302  if strings[0] in a:
303  try:
304  return self.__findInJson(a[strings[0]], strings[1:])
305  except (TypeError, KeyError): #TypeError because a could be a string and contain strings[0]
306  pass
307  #if it's not found
308  raise KeyError("Can't find " + strings[0])
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:287
def dataset.Dataset.__getData (   self,
  dasQuery,
  dasLimit = 0 
)
private

Definition at line 339 of file dataset.py.

Referenced by dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.__getParentDataset(), dataset.Dataset.__getRunList(), and dataset.Dataset.convertTimeToRun().

339 
def __getData( self, dasQuery, dasLimit = 0 ):
def dataset.Dataset.__getDataType (   self)
private

Definition at line 371 of file dataset.py.

References dataset.Dataset.__filename, dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__name, dataset.Dataset.__predefined, ElectronMVAID.ElectronMVAID.name, counter.Counter.name, entry.name, average.Average.name, histograms.Histograms.name, geometrydata.GeometryData.name, cond::persistency::TAG::NAME.name, TmModule.name, core.autovars.NTupleVariable.name, cond::persistency::GLOBAL_TAG::NAME.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::TAG::TIME_TYPE.name, genericValidation.GenericValidation.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, cond::persistency::TAG::OBJECT_TYPE.name, ora::RecordSpecImpl::Item.name, preexistingValidation.PreexistingValidation.name, cond::persistency::GLOBAL_TAG::RELEASE.name, cond::persistency::TAG::SYNCHRONIZATION.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, MEPSet.name, cond::persistency::TAG::END_OF_VALIDITY.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::TAG::DESCRIPTION.name, cond::persistency::GTEditorData.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, FWTGeoRecoGeometry::Info.name, Types._Untracked.name, cond::persistency::TAG::INSERTION_TIME.name, cond::persistency::TAG::MODIFICATION_TIME.name, dataset.BaseDataset.name, OutputMEPSet.name, personalPlayback.Applet.name, ParameterSet.name, PixelDCSObject< class >::Item.name, analyzer.Analyzer.name, DQMRivetClient::LumiOption.name, MagCylinder.name, alignment.Alignment.name, ParSet.name, DQMRivetClient::ScaleFactorOption.name, SingleObjectCondition.name, EgHLTOfflineSummaryClient::SumHistBinData.name, cond::persistency::GTProxyData.name, o2o.O2OJob.name, DQMGenericClient::EfficOption.name, XMLRBXPedestalsLoader::_loaderBaseConfig.name, XMLHTRZeroSuppressionLoader::_loaderBaseConfig.name, core.autovars.NTupleObjectType.name, MyWatcher.name, edm::PathTimingSummary.name, cond::TimeTypeSpecs.name, lumi::TriggerInfo.name, edm::PathSummary.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, perftools::EdmEventSize::BranchRecord.name, PixelEndcapLinkMaker::Item.name, FWTableViewManager::TableEntry.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, PixelBarrelLinkMaker::Item.name, Mapper::definition< ScannerT >.name, EcalLogicID.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, ExpressionHisto< T >.name, XMLProcessor::_loaderBaseConfig.name, DQMGenericClient::ProfileOption.name, TreeCrawler.Package.name, cond::persistency::PAYLOAD::HASH.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, cond::persistency::PAYLOAD::DATA.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, MagGeoBuilderFromDDD::volumeHandle.name, options.ConnectionHLTMenu.name, cond::persistency::PAYLOAD::VERSION.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, DQMGenericClient::NormOption.name, Node.name, DQMGenericClient::CDOption.name, FastHFShowerLibrary.name, h4DSegm.name, PhysicsTools::Calibration::Variable.name, cond::TagInfo_t.name, CounterChecker.name, EDMtoMEConverter.name, looper.Looper.name, MEtoEDM< T >::MEtoEDMObject.name, cond::persistency::IOV::TAG_NAME.name, cond::persistency::IOV::SINCE.name, TrackerSectorStruct.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, classes.MonitorData.name, MuonGeometrySanityCheckPoint.name, classes.OutputData.name, options.HLTProcessOptions.name, h2DSegm.name, config.Analyzer.name, core.autovars.NTupleSubObject.name, DQMNet::WaitObject.name, AlpgenParameterName.name, SiStripMonitorDigi.name, core.autovars.NTupleObject.name, geometry.Structure.name, cond::persistency::TAG_LOG::TAG_NAME.name, cond::persistency::TAG_LOG::EVENT_TIME.name, cond::persistency::TAG_LOG::USER_NAME.name, cond::persistency::TAG_LOG::HOST_NAME.name, cond::persistency::TAG_LOG::COMMAND.name, cond::persistency::TAG_LOG::ACTION.name, cond::persistency::TAG_LOG::USER_TEXT.name, config.Service.name, core.autovars.NTupleCollection.name, FastTimerService::LuminosityDescription.name, personalPlayback.FrameworkJob.name, plotscripts.SawTeethFunction.name, FastTimerService::ProcessDescription.name, hTMaxCell.name, cscdqm::ParHistoDef.name, BeautifulSoup.Tag.name, SummaryOutputProducer::GenericSummary.name, TiXmlAttribute.name, BeautifulSoup.SoupStrainer.name, and python.rootplot.root2matplotlib.replace().

Referenced by dataset.Dataset.dataType().

371 
372  def __getDataType( self ):
373  if self.__predefined:
374  with open(self.__filename) as f:
375  datatype = None
376  for line in f.readlines():
377  if line.startswith("#data type: "):
378  if datatype is not None:
379  raise AllInOneError(self.__filename + " has multiple 'data type' lines.")
380  datatype = line.replace("#data type: ", "").replace("\n","")
381  return datatype
382  return "unknown"
383 
384  dasQuery_type = ( 'dataset dataset=%s | grep dataset.datatype,'
385  'dataset.name'%( self.__name ) )
386  data = self.__getData( dasQuery_type )
387 
388  try:
389  return self.__findInJson(data, ["dataset", "datatype"])
390  except KeyError:
391  print ("Cannot find the datatype of the dataset '%s'\n"
392  "It may not be possible to automatically find the magnetic field,\n"
393  "and you will not be able run in CRAB mode"
394  %( self.name() ))
395  return "unknown"
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:287
dataset.Dataset.__getDataType
def __getDataType
Definition: dataset.py:371
def dataset.Dataset.__getFileInfoList (   self,
  dasLimit,
  parent = False 
)
private

Definition at line 531 of file dataset.py.

References dataset.Dataset.__fileInfoList, dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__name, dataset.Dataset.__parentFileInfoList, dataset.Dataset.__predefined, ElectronMVAID.ElectronMVAID.name, counter.Counter.name, entry.name, average.Average.name, histograms.Histograms.name, geometrydata.GeometryData.name, cond::persistency::GLOBAL_TAG::NAME.name, cond::persistency::TAG::NAME.name, TmModule.name, core.autovars.NTupleVariable.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::TAG::TIME_TYPE.name, genericValidation.GenericValidation.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, cond::persistency::TAG::OBJECT_TYPE.name, ora::RecordSpecImpl::Item.name, cond::persistency::GLOBAL_TAG::RELEASE.name, preexistingValidation.PreexistingValidation.name, cond::persistency::TAG::SYNCHRONIZATION.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, MEPSet.name, cond::persistency::TAG::END_OF_VALIDITY.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::TAG::DESCRIPTION.name, cond::persistency::GTEditorData.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, FWTGeoRecoGeometry::Info.name, Types._Untracked.name, cond::persistency::TAG::INSERTION_TIME.name, dataset.BaseDataset.name, cond::persistency::TAG::MODIFICATION_TIME.name, OutputMEPSet.name, personalPlayback.Applet.name, ParameterSet.name, PixelDCSObject< class >::Item.name, analyzer.Analyzer.name, DQMRivetClient::LumiOption.name, MagCylinder.name, alignment.Alignment.name, ParSet.name, DQMRivetClient::ScaleFactorOption.name, SingleObjectCondition.name, EgHLTOfflineSummaryClient::SumHistBinData.name, o2o.O2OJob.name, DQMGenericClient::EfficOption.name, XMLHTRZeroSuppressionLoader::_loaderBaseConfig.name, XMLRBXPedestalsLoader::_loaderBaseConfig.name, core.autovars.NTupleObjectType.name, cond::persistency::GTProxyData.name, MyWatcher.name, edm::PathTimingSummary.name, cond::TimeTypeSpecs.name, lumi::TriggerInfo.name, edm::PathSummary.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, PixelEndcapLinkMaker::Item.name, perftools::EdmEventSize::BranchRecord.name, FWTableViewManager::TableEntry.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, PixelBarrelLinkMaker::Item.name, Mapper::definition< ScannerT >.name, EcalLogicID.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, ExpressionHisto< T >.name, XMLProcessor::_loaderBaseConfig.name, cond::persistency::PAYLOAD::HASH.name, DQMGenericClient::ProfileOption.name, TreeCrawler.Package.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, cond::persistency::PAYLOAD::DATA.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, MagGeoBuilderFromDDD::volumeHandle.name, options.ConnectionHLTMenu.name, cond::persistency::PAYLOAD::VERSION.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, DQMGenericClient::NormOption.name, Node.name, DQMGenericClient::CDOption.name, FastHFShowerLibrary.name, h4DSegm.name, PhysicsTools::Calibration::Variable.name, cond::TagInfo_t.name, CounterChecker.name, EDMtoMEConverter.name, looper.Looper.name, MEtoEDM< T >::MEtoEDMObject.name, cond::persistency::IOV::TAG_NAME.name, TrackerSectorStruct.name, cond::persistency::IOV::SINCE.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, classes.MonitorData.name, MuonGeometrySanityCheckPoint.name, classes.OutputData.name, options.HLTProcessOptions.name, h2DSegm.name, config.Analyzer.name, core.autovars.NTupleSubObject.name, DQMNet::WaitObject.name, AlpgenParameterName.name, SiStripMonitorDigi.name, core.autovars.NTupleObject.name, geometry.Structure.name, cond::persistency::TAG_LOG::TAG_NAME.name, cond::persistency::TAG_LOG::EVENT_TIME.name, cond::persistency::TAG_LOG::USER_NAME.name, cond::persistency::TAG_LOG::HOST_NAME.name, cond::persistency::TAG_LOG::COMMAND.name, cond::persistency::TAG_LOG::ACTION.name, cond::persistency::TAG_LOG::USER_TEXT.name, config.Service.name, core.autovars.NTupleCollection.name, FastTimerService::LuminosityDescription.name, personalPlayback.FrameworkJob.name, plotscripts.SawTeethFunction.name, FastTimerService::ProcessDescription.name, hTMaxCell.name, cscdqm::ParHistoDef.name, BeautifulSoup.Tag.name, SummaryOutputProducer::GenericSummary.name, TiXmlAttribute.name, BeautifulSoup.SoupStrainer.name, and dataset.Dataset.parentDataset().

Referenced by dataset.Dataset.fileInfoList().

531 
532  def __getFileInfoList( self, dasLimit, parent = False ):
533  if self.__predefined:
534  if parent:
535  extendstring = "secFiles.extend"
536  else:
537  extendstring = "readFiles.extend"
538  with open(self.__fileName) as f:
539  files = []
540  copy = False
541  for line in f.readlines():
542  if "]" in line:
543  copy = False
544  if copy:
545  files.append({name: line.translate(None, "', " + '"')})
546  if extendstring in line and "[" in line and "]" not in line:
547  copy = True
548  return files
549 
550  if self.__fileInfoList and not parent:
551  return self.__fileInfoList
552  if self.__parentFileInfoList and parent:
553  return self.__parentFileInfoList
554 
555  if parent:
556  searchdataset = self.parentDataset()
557  else:
558  searchdataset = self.__name
559  dasQuery_files = ( 'file dataset=%s | grep file.name, file.nevents, '
560  'file.creation_time, '
561  'file.modification_time'%( searchdataset ) )
562  print "Requesting file information for '%s' from DAS..."%( searchdataset ),
563  data = self.__getData( dasQuery_files, dasLimit )
564  print "Done."
565  data = [ self.__findInJson(entry,"file") for entry in data ]
566  if len( data ) == 0:
567  msg = ("No files are available for the dataset '%s'. This can be "
568  "due to a typo or due to a DAS problem. Please check the "
569  "spelling of the dataset and/or retry to run "
570  "'validateAlignments.py'."%( self.name() ))
571  raise AllInOneError( msg )
572  fileInformationList = []
573  for file in data:
574  fileName = 'unknown'
575  try:
576  fileName = self.__findInJson(file, "name")
577  fileCreationTime = self.__findInJson(file, "creation_time")
578  fileNEvents = self.__findInJson(file, "nevents")
579  except KeyError:
580  print ("DAS query gives bad output for file '%s'. Skipping it.\n"
581  "It may work if you try again later.") % fileName
582  fileNEvents = 0
583  # select only non-empty files
584  if fileNEvents == 0:
585  continue
586  fileDict = { "name": fileName,
587  "creation_time": fileCreationTime,
588  "nevents": fileNEvents
589  }
590  fileInformationList.append( fileDict )
591  fileInformationList.sort( key=lambda info: self.__findInJson(info,"name") )
592  if parent:
593  self.__parentFileInfoList = fileInformationList
594  else:
595  self.__fileInfoList = fileInformationList
596  return fileInformationList
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:287
dataset.Dataset.__getFileInfoList
def __getFileInfoList
Definition: dataset.py:531
dataset.Dataset.parentDataset
def parentDataset
Definition: dataset.py:702
def dataset.Dataset.__getMagneticField (   self)
private

Definition at line 406 of file dataset.py.

References dataset.Dataset.__cmssw, dataset.Dataset.__cmsswrelease, dataset.Dataset.__dataType, dataset.Dataset.__filename, dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__name, dataset.Dataset.__predefined, and python.rootplot.root2matplotlib.replace().

Referenced by dataset.Dataset.magneticField().

406 
407  def __getMagneticField( self ):
408  Bfieldlocation = os.path.join( self.__cmssw, "python", "Configuration", "StandardSequences" )
409  if not os.path.isdir(Bfieldlocation):
410  Bfieldlocation = os.path.join( self.__cmsswrelease, "python", "Configuration", "StandardSequences" )
411  Bfieldlist = [ f.replace("_cff.py",'') \
412  for f in os.listdir(Bfieldlocation) \
413  if f.startswith("MagneticField_") and f.endswith("_cff.py") ]
414  Bfieldlist.sort( key = lambda Bfield: -len(Bfield) ) #Put it in order of decreasing length, so that searching in the name gives the longer match
415 
416  if self.__predefined:
417  with open(self.__filename) as f:
418  datatype = None
419  Bfield = None
420  for line in f.readlines():
421  if line.startswith("#data type: "):
422  if datatype is not None:
423  raise AllInOneError(self.__filename + " has multiple 'data type' lines.")
424  datatype = line.replace("#data type: ", "").replace("\n","")
425  datatype = datatype.split("#")[0].strip()
426  if line.startswith("#magnetic field: "):
427  if Bfield is not None:
428  raise AllInOneError(self.__filename + " has multiple 'magnetic field' lines.")
429  Bfield = line.replace("#magnetic field: ", "").replace("\n","")
430  Bfield = Bfield.split("#")[0].strip()
431  if Bfield is not None:
432  Bfield = Bfield.split(",")[0]
433  if Bfield in Bfieldlist or Bfield == "unknown":
434  return Bfield
435  else:
436  print "Your dataset has magnetic field '%s', which does not exist in your CMSSW version!" % Bfield
437  print "Using Bfield='unknown' - this will revert to the default"
438  return "unknown"
439  elif datatype == "data":
440  return "MagneticField" #this should be in the "#magnetic field" line, but for safety in case it got messed up
441  else:
442  return "unknown"
443 
444  if self.__dataType == "data":
445  return "MagneticField"
446 
447  dasQuery_B = ( 'dataset dataset=%s'%( self.__name ) ) #try to find the magnetic field from DAS
448  data = self.__getData( dasQuery_B ) #it seems to be there for the newer (7X) MC samples, except cosmics
449 
450  try:
451  Bfield = self.__findInJson(data, ["dataset", "mcm", "sequences", "magField"])
452  if Bfield in Bfieldlist:
453  return Bfield
454  elif Bfield == "38T" or Bfield == "38T_PostLS1":
455  return "MagneticField"
456  elif "MagneticField_" + Bfield in Bfieldlist:
457  return "MagneticField_" + Bfield
458  elif Bfield == "":
459  pass
460  else:
461  print "Your dataset has magnetic field '%s', which does not exist in your CMSSW version!" % Bfield
462  print "Using Bfield='unknown' - this will revert to the default magnetic field"
463  return "unknown"
464  except KeyError:
465  pass
466 
467  for possibleB in Bfieldlist:
468  if (possibleB != "MagneticField"
469  and possibleB.replace("MagneticField_","") in self.__name.replace("TkAlCosmics0T", "")):
470  #final attempt - try to identify the dataset from the name
471  #all cosmics dataset names contain "TkAlCosmics0T"
472  if possibleB == "MagneticField_38T" or possibleB == "MagneticField_38T_PostLS1":
473  return "MagneticField"
474  return possibleB
475 
476  return "unknown"
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:287
dataset.Dataset.__getMagneticField
def __getMagneticField
Definition: dataset.py:406
def dataset.Dataset.__getMagneticFieldForRun (   self,
  run = -1,
  tolerance = 0.5 
)
private 
For MC, this returns the same as the previous function.
   For data, it gets the magnetic field from the runs.  This is important for
   deciding which template to use for offlinevalidation

Definition at line 477 of file dataset.py.

References dataset.Dataset.__dataType, dataset.Dataset.__filename, dataset.Dataset.__findInJson(), dataset.Dataset.__firstusedrun, dataset.Dataset.__getData(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.__lastusedrun, dataset.Dataset.__magneticField, dataset.Dataset.__name, dataset.Dataset.__predefined, funct.abs(), python.rootplot.root2matplotlib.replace(), and split.

Referenced by dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.dump_cff(), and dataset.Dataset.magneticFieldForRun().

477 
478  def __getMagneticFieldForRun( self, run = -1, tolerance = 0.5 ):
479  """For MC, this returns the same as the previous function.
480  For data, it gets the magnetic field from the runs. This is important for
481  deciding which template to use for offlinevalidation
482  """
483  if self.__dataType == "mc" and self.__magneticField == "MagneticField":
484  return 3.8 #For 3.8T MC the default MagneticField is used
485  if "T" in self.__magneticField:
486  Bfield = self.__magneticField.split("T")[0].replace("MagneticField_","")
487  try:
488  return float(Bfield) / 10.0 #e.g. 38T and 38T_PostLS1 both return 3.8
489  except ValueError:
490  pass
491  if self.__predefined:
492  with open(self.__filename) as f:
493  Bfield = None
494  for line in f.readlines():
495  if line.startswith("#magnetic field: ") and "," in line:
496  if Bfield is not None:
497  raise AllInOneError(self.__filename + " has multiple 'magnetic field' lines.")
498  return float(line.replace("#magnetic field: ", "").split(",")[1].split("#")[0].strip())
499 
500  if run > 0:
501  dasQuery = ('run = %s'%run) #for data
502  data = self.__getData(dasQuery)
503  try:
504  return self.__findInJson(data, ["run","bfield"])
505  except KeyError:
506  return "unknown Can't get the magnetic field for run %s from DAS" % run
507 
508  #run < 0 - find B field for the first and last runs, and make sure they're compatible
509  # (to within tolerance)
510  #NOT FOOLPROOF! The magnetic field might go up and then down, or vice versa
511  if self.__firstusedrun is None or self.__lastusedrun is None:
512  return "unknown Can't get the exact magnetic field for the dataset until data has been retrieved from DAS."
513  firstrunB = self.__getMagneticFieldForRun(self.__firstusedrun)
514  lastrunB = self.__getMagneticFieldForRun(self.__lastusedrun)
515  try:
516  if abs(firstrunB - lastrunB) <= tolerance:
517  return .5*(firstrunB + lastrunB)
518  print firstrunB, lastrunB, tolerance
519  return ("unknown The beginning and end of your run range for %s\n"
520  "have different magnetic fields (%s, %s)!\n"
521  "Try limiting the run range using firstRun, lastRun, begin, end, or JSON,\n"
522  "or increasing the tolerance (in dataset.py) from %s.") % (self.__name, firstrunB, lastrunB, tolerance)
523  except TypeError:
524  try:
525  if "unknown" in firstrunB:
526  return firstrunB
527  else:
528  return lastrunB
529  except TypeError:
530  return lastrunB
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:287
dataset.Dataset.__getMagneticFieldForRun
def __getMagneticFieldForRun
Definition: dataset.py:477
funct::abs
Abs< T >::type abs(const T &t)
Definition: Abs.h:22
split
double split
Definition: MVATrainer.cc:139
def dataset.Dataset.__getParentDataset (   self)
private

Definition at line 396 of file dataset.py.

References dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), and dataset.Dataset.__name.

Referenced by dataset.Dataset.parentDataset().

396 
397  def __getParentDataset( self ):
398  dasQuery = "parent dataset=" + self.__name
399  data = self.__getData( dasQuery )
400  try:
401  return self.__findInJson(data, ["parent", "name"])
402  except KeyError:
403  raise AllInOneError("Cannot find the parent of the dataset '" + self.__name + "'\n"
404  "Here is the DAS output:\n" + str(jsondict) +
405  "\nIt's possible that this was a server error. If so, it may work if you try again later")
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:287
dataset.Dataset.__getParentDataset
def __getParentDataset
Definition: dataset.py:396
def dataset.Dataset.__getRunList (   self)
private

Definition at line 597 of file dataset.py.

References dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__name, and dataset.Dataset.__runList.

Referenced by dataset.Dataset.__createSnippet(), dataset.Dataset.convertTimeToRun(), and dataset.Dataset.runList().

597 
598  def __getRunList( self ):
599  if self.__runList:
600  return self.__runList
601  dasQuery_runs = ( 'run dataset=%s | grep run.run_number,'
602  'run.creation_time'%( self.__name ) )
603  print "Requesting run information for '%s' from DAS..."%( self.__name ),
604  data = self.__getData( dasQuery_runs )
605  print "Done."
606  data = [ self.__findInJson(entry,"run") for entry in data ]
607  data.sort( key = lambda run: self.__findInJson(run, "run_number") )
608  self.__runList = data
609  return data
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:287
dataset.Dataset.__getRunList
def __getRunList
Definition: dataset.py:597
def dataset.Dataset.buildListOfBadFiles (   self) 
fills the list of bad files from the IntegrityCheck log.

When the integrity check file is not available,
files are considered as good.

Definition at line 275 of file dataset.py.

275 
276  def buildListOfBadFiles(self):
277  '''fills the list of bad files from the IntegrityCheck log.
278 
279  When the integrity check file is not available,
280  files are considered as good.'''
281  mask = "IntegrityCheck"
282 
283  self.bad_files = {}
284  self.good_files = []
285 
286  file_mask = castortools.matchingFiles(self.castorDir, '^%s_.*\.txt$' % mask)
287  if file_mask:
288  # here to avoid circular dependency
289  from edmIntegrityCheck import PublishToFileSystem
290  p = PublishToFileSystem(mask)
291  report = p.get(self.castorDir)
292  if report is not None and report:
293  self.maskExists = True
294  self.report = report
295  dup = report.get('ValidDuplicates',{})
296  for name, status in report['Files'].iteritems():
297  # print name, status
298  if not status[0]:
299  self.bad_files[name] = 'MarkedBad'
300  elif name in dup:
301  self.bad_files[name] = 'ValidDup'
302  else:
303  self.good_files.append( name )
304  else:
305  raise IntegrityCheckError( "ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found" )
dataset.Dataset.buildListOfBadFiles
def buildListOfBadFiles
Definition: dataset.py:275
def dataset.Dataset.buildListOfFiles (   self,
  pattern = '.*root' 
) 
fills list of files, taking all root files matching the pattern in the castor dir

Definition at line 271 of file dataset.py.

271 
272  def buildListOfFiles(self, pattern='.*root'):
273  '''fills list of files, taking all root files matching the pattern in the castor dir'''
274  self.files = castortools.matchingFiles( self.castorDir, pattern )
dataset.Dataset.buildListOfFiles
def buildListOfFiles
Definition: dataset.py:271
def dataset.Dataset.convertTimeToRun (   self,
  begin = None,
  end = None,
  firstRun = None,
  lastRun = None,
  shortTuple = True 
)

Definition at line 624 of file dataset.py.

References dataset.Dataset.__dateString(), dataset.Dataset.__datetime(), dataset.Dataset.__find_ge(), dataset.Dataset.__find_lt(), dataset.Dataset.__findInJson(), dataset.Dataset.__getData(), dataset.Dataset.__getRunList(), and dataset.Dataset.__name.

Referenced by dataset.Dataset.__createSnippet(), and dataset.Dataset.__dateString().

624 
625  shortTuple = True ):
626  if ( begin and firstRun ) or ( end and lastRun ):
627  msg = ( "The Usage of "
628  + "'begin' & 'firstRun' " * int( bool( begin and
629  firstRun ) )
630  + "and " * int( bool( ( begin and firstRun ) and
631  ( end and lastRun ) ) )
632  + "'end' & 'lastRun' " * int( bool( end and lastRun ) )
633  + "is ambigous." )
634  raise AllInOneError( msg )
635 
636  if begin or end:
637  runList = [ self.__findInJson(run, "run_number") for run in self.__getRunList() ]
638 
639  if begin:
640  lastdate = begin
641  for delta in [ 1, 5, 10, 20, 30 ]: #try searching for about 2 months after begin
642  firstdate = lastdate
643  lastdate = self.__dateString(self.__datetime(firstdate) + datetime.timedelta(delta))
644  dasQuery_begin = "run date between[%s,%s]" % (firstdate, lastdate)
645  begindata = self.__getData(dasQuery_begin)
646  if len(begindata) > 0:
647  begindata.sort(key = lambda run: self.__findInJson(run, ["run", "run_number"]))
648  try:
649  runIndex = self.__find_ge( runList, self.__findInJson(begindata[0], ["run", "run_number"]))
650  except ValueError:
651  msg = ( "Your 'begin' is after the creation time of the last "
652  "run in the dataset\n'%s'"%( self.__name ) )
653  raise AllInOneError( msg )
654  firstRun = runList[runIndex]
655  begin = None
656  break
657 
658  if begin:
659  raise AllInOneError("No runs within a reasonable time interval after your 'begin'."
660  "Try using a 'begin' that has runs soon after it (within 2 months at most)")
661 
662  if end:
663  firstdate = end
664  for delta in [ 1, 5, 10, 20, 30 ]: #try searching for about 2 months before end
665  lastdate = firstdate
666  firstdate = self.__dateString(self.__datetime(lastdate) - datetime.timedelta(delta))
667  dasQuery_end = "run date between[%s,%s]" % (firstdate, lastdate)
668  enddata = self.__getData(dasQuery_end)
669  if len(enddata) > 0:
670  enddata.sort(key = lambda run: self.__findInJson(run, ["run", "run_number"]))
671  try:
672  runIndex = self.__find_lt( runList, self.__findInJson(enddata[-1], ["run", "run_number"]))
673  except ValueError:
674  msg = ( "Your 'end' is before the creation time of the first "
675  "run in the dataset\n'%s'"%( self.__name ) )
676  raise AllInOneError( msg )
677  lastRun = runList[runIndex]
678  end = None
679  break
680 
681  if end:
682  raise AllInOneError("No runs within a reasonable time interval before your 'end'."
683  "Try using an 'end' that has runs soon before it (within 2 months at most)")
684 
685  if shortTuple:
686  return firstRun, lastRun
687  else:
688  return begin, end, firstRun, lastRun
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:287
dataset.Dataset.__getRunList
def __getRunList
Definition: dataset.py:597
dataset.Dataset.__dateString
def __dateString
Definition: dataset.py:619
def dataset.Dataset.datasetSnippet (   self,
  jsonPath = None,
  begin = None,
  end = None,
  firstRun = None,
  lastRun = None,
  crab = False,
  parent = False 
)

Definition at line 708 of file dataset.py.

References dataset.Dataset.__createSnippet(), dataset.Dataset.__filename, dataset.Dataset.__name, dataset.Dataset.__official, dataset.Dataset.__origName, dataset.Dataset.__predefined, and dataset.Dataset.dump_cff().

Referenced by dataset.Dataset.parentDataset().

708 
709  firstRun = None, lastRun = None, crab = False, parent = False ):
710  if self.__predefined and parent:
711  with open(self.__filename) as f:
712  if "secFiles.extend" not in f.read():
713  msg = ("The predefined dataset '%s' does not contain secondary files, "
714  "which your validation requires!") % self.__name
715  if self.__official:
716  self.__name = self.__origName
717  self.__predefined = False
718  print msg
719  print ("Retreiving the files from DAS. You will be asked if you want "
720  "to overwrite the old dataset.\n"
721  "It will still be compatible with validations that don't need secondary files.")
722  else:
723  raise AllInOneError(msg)
724 
725  if self.__predefined:
726  snippet = ("process.load(\"Alignment.OfflineValidation.%s_cff\")\n"
727  "process.maxEvents = cms.untracked.PSet(\n"
728  " input = cms.untracked.int32(.oO[nEvents]Oo. / .oO[parallelJobs]Oo.)\n"
729  ")\n"
730  "process.source.skipEvents=cms.untracked.uint32(.oO[nIndex]Oo.*.oO[nEvents]Oo./.oO[parallelJobs]Oo.)"
731  %(self.__name))
732  if not parent:
733  with open(self.__filename) as f:
734  if "secFiles.extend" in f.read():
735  snippet += "\nprocess.source.secondaryFileNames = cms.untracked.vstring()"
736  return snippet
737  theMap = { "process": "process.",
738  "tab": " " * len( "process." ),
739  "nEvents": ".oO[nEvents]Oo. / .oO[parallelJobs]Oo.",
740  "skipEventsString": "process.source.skipEvents=cms.untracked.uint32(.oO[nIndex]Oo.*.oO[nEvents]Oo./.oO[parallelJobs]Oo.)\n",
741  "importCms": "",
742  "header": ""
743  }
744  datasetSnippet = self.__createSnippet( jsonPath = jsonPath,
745  begin = begin,
746  end = end,
747  firstRun = firstRun,
748  lastRun = lastRun,
749  repMap = theMap,
750  crab = crab,
751  parent = parent )
752  if jsonPath == "" and begin == "" and end == "" and firstRun == "" and lastRun == "":
753  try:
754  self.dump_cff(parent = parent)
755  except AllInOneError as e:
756  print "Can't store the dataset as a cff:"
757  print e
758  print "This may be inconvenient in the future, but will not cause a problem for this validation."
759  return datasetSnippet
dataset.Dataset.__createSnippet
def __createSnippet
Definition: dataset.py:117
def dataset.Dataset.dataType (   self)

Definition at line 689 of file dataset.py.

References dataset.Dataset.__dataType, and dataset.Dataset.__getDataType().

689 
690  def dataType( self ):
691  if not self.__dataType:
692  self.__dataType = self.__getDataType()
693  return self.__dataType
dataset.Dataset.__getDataType
def __getDataType
Definition: dataset.py:371
def dataset.Dataset.dump_cff (   self,
  outName = None,
  jsonPath = None,
  begin = None,
  end = None,
  firstRun = None,
  lastRun = None,
  parent = False 
)

Definition at line 761 of file dataset.py.

References dataset.Dataset.__alreadyStored, dataset.Dataset.__cmssw, dataset.Dataset.__createSnippet(), dataset.Dataset.__dataType, dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.__magneticField, dataset.Dataset.__name, python.rootplot.root2matplotlib.replace(), and split.

Referenced by dataset.Dataset.datasetSnippet().

761 
762  end = None, firstRun = None, lastRun = None, parent = False ):
763  if self.__alreadyStored:
764  return
765  self.__alreadyStored = True
766  if outName == None:
767  outName = "Dataset" + self.__name.replace("/", "_")
768  packageName = os.path.join( "Alignment", "OfflineValidation" )
769  if not os.path.exists( os.path.join(
770  self.__cmssw, "src", packageName ) ):
771  msg = ("You try to store the predefined dataset'%s'.\n"
772  "For that you need to check out the package '%s' to your "
773  "private relase area in\n"%( outName, packageName )
774  + self.__cmssw )
775  raise AllInOneError( msg )
776  theMap = { "process": "",
777  "tab": "",
778  "nEvents": str( -1 ),
779  "skipEventsString": "",
780  "importCms": "import FWCore.ParameterSet.Config as cms\n",
781  "header": "#Do not delete or (unless you know what you're doing) change these comments\n"
782  "#%(name)s\n"
783  "#data type: %(dataType)s\n"
784  "#magnetic field: .oO[magneticField]Oo.\n" #put in magnetic field later
785  %{"name": self.__name, #need to create the snippet before getting the magnetic field
786  "dataType": self.__dataType} #so that we know the first and last runs
787  }
788  dataset_cff = self.__createSnippet( jsonPath = jsonPath,
789  begin = begin,
790  end = end,
791  firstRun = firstRun,
792  lastRun = lastRun,
793  repMap = theMap,
794  parent = parent)
795  magneticField = self.__magneticField
796  if magneticField == "MagneticField":
797  magneticField = "%s, %s #%s" % (magneticField,
798  str(self.__getMagneticFieldForRun()).replace("\n"," ").split("#")[0].strip(),
799  "Use MagneticField_cff.py; the number is for determining which track selection to use."
800  )
801  dataset_cff = dataset_cff.replace(".oO[magneticField]Oo.",magneticField)
802  filePath = os.path.join( self.__cmssw, "src", packageName,
803  "python", outName + "_cff.py" )
804  if os.path.exists( filePath ):
805  existMsg = "The predefined dataset '%s' already exists.\n"%( outName )
806  askString = "Do you want to overwrite it? [y/n]\n"
807  inputQuery = existMsg + askString
808  while True:
809  userInput = raw_input( inputQuery ).lower()
810  if userInput == "y":
811  break
812  elif userInput == "n":
813  return
814  else:
815  inputQuery = askString
816  print ( "The predefined dataset '%s' will be stored in the file\n"
817  %( outName )
818  + filePath +
819  "\nFor future use you have to do 'scram b'." )
820  print
821  theFile = open( filePath, "w" )
822  theFile.write( dataset_cff )
823  theFile.close()
824  return
dataset.Dataset.__getMagneticFieldForRun
def __getMagneticFieldForRun
Definition: dataset.py:477
dataset.Dataset.__createSnippet
def __createSnippet
Definition: dataset.py:117
split
double split
Definition: MVATrainer.cc:139
def dataset.Dataset.extractFileSizes (   self) 
Get the file size for each file, from the eos ls -l command.

Definition at line 306 of file dataset.py.

References dataset.EOSDataset.castorDir, and dataset.Dataset.castorDir.

306 
307  def extractFileSizes(self):
308  '''Get the file size for each file, from the eos ls -l command.'''
309  # EOS command does not work in tier3
310  lsout = castortools.runXRDCommand(self.castorDir,'dirlist')[0]
311  lsout = lsout.split('\n')
312  self.filesAndSizes = {}
313  for entry in lsout:
314  values = entry.split()
315  if( len(values) != 5):
316  continue
317  # using full abs path as a key.
318  file = '/'.join([self.lfnDir, values[4].split("/")[-1]])
319  size = values[1]
320  self.filesAndSizes[file] = size
dataset.Dataset.extractFileSizes
def extractFileSizes
Definition: dataset.py:306
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
reco::if
if(dp >Float(M_PI)) dp-
split
double split
Definition: MVATrainer.cc:139
def dataset.Dataset.fileInfoList (   self,
  parent = False 
)

Definition at line 840 of file dataset.py.

References dataset.Dataset.__dasLimit, and dataset.Dataset.__getFileInfoList().

Referenced by dataset.Dataset.fileList().

840 
841  def fileInfoList( self, parent = False ):
842  return self.__getFileInfoList( self.__dasLimit, parent )
dataset.Dataset.__getFileInfoList
def __getFileInfoList
Definition: dataset.py:531
dataset.Dataset.fileInfoList
def fileInfoList
Definition: dataset.py:840
def dataset.Dataset.fileList (   self,
  parent = False 
)

Definition at line 825 of file dataset.py.

References dataset.Dataset.__fileList, dataset.Dataset.__findInJson(), dataset.Dataset.__parentFileList, and dataset.Dataset.fileInfoList().

Referenced by dataset.Dataset.__createSnippet().

825 
826  def fileList( self, parent = False ):
827  if self.__fileList and not parent:
828  return self.__fileList
829  if self.__parentFileList and parent:
830  return self.__parentFileList
831 
832  fileList = [ self.__findInJson(fileInfo,"name") \
833  for fileInfo in self.fileInfoList(parent) ]
834 
835  if not parent:
836  self.__fileList = fileList
837  else:
838  self.__parentFileList = fileList
839  return fileList
dataset.Dataset.__findInJson
def __findInJson
Definition: dataset.py:287
dataset.Dataset.fileInfoList
def fileInfoList
Definition: dataset.py:840
def dataset.Dataset.forcerunrange (   self,
  firstRun,
  lastRun,
  s 
) 
s must be in the format run1:lum1-run2:lum2

Definition at line 309 of file dataset.py.

References dataset.Dataset.__firstusedrun, dataset.Dataset.__lastusedrun, and split.

Referenced by dataset.Dataset.getForceRunRangeFunction().

309 
310  def forcerunrange(self, firstRun, lastRun, s):
311  """s must be in the format run1:lum1-run2:lum2"""
312  s = s.group()
313  run1 = s.split("-")[0].split(":")[0]
314  lum1 = s.split("-")[0].split(":")[1]
315  try:
316  run2 = s.split("-")[1].split(":")[0]
317  lum2 = s.split("-")[1].split(":")[1]
318  except IndexError:
319  run2 = run1
320  lum2 = lum1
321  if int(run2) < firstRun or int(run1) > lastRun:
322  return ""
323  if int(run1) < firstRun or firstRun < 0:
324  run1 = firstRun
325  lum1 = 1
326  if int(run2) > lastRun:
327  run2 = lastRun
328  lum2 = "max"
329  if int(run1) < self.__firstusedrun or self.__firstusedrun < 0:
330  self.__firstusedrun = int(run1)
331  if int(run2) > self.__lastusedrun:
332  self.__lastusedrun = int(run2)
333  return "%s:%s-%s:%s" % (run1, lum1, run2, lum2)
dataset.Dataset.forcerunrange
def forcerunrange
Definition: dataset.py:309
split
double split
Definition: MVATrainer.cc:139
def dataset.Dataset.getForceRunRangeFunction (   self,
  firstRun,
  lastRun 
)

Definition at line 334 of file dataset.py.

References dataset.Dataset.forcerunrange().

Referenced by dataset.Dataset.__createSnippet().

334 
335  def getForceRunRangeFunction(self, firstRun, lastRun):
336  def forcerunrangefunction(s):
337  return self.forcerunrange(firstRun, lastRun, s)
338  return forcerunrangefunction
dataset.Dataset.forcerunrange
def forcerunrange
Definition: dataset.py:309
dataset.Dataset.getForceRunRangeFunction
def getForceRunRangeFunction
Definition: dataset.py:334
def dataset.Dataset.getPrimaryDatasetEntries (   self)

Definition at line 326 of file dataset.py.

References runall.testit.report, dataset.BaseDataset.report, ALIUtils.report, and WorkFlowRunner.WorkFlowRunner.report.

326 
327  def getPrimaryDatasetEntries(self):
328  if self.report is not None and self.report:
329  return int(self.report.get('PrimaryDatasetEntries',-1))
330  return -1
331 
dataset.Dataset.getPrimaryDatasetEntries
def getPrimaryDatasetEntries
Definition: dataset.py:326
def dataset.Dataset.magneticField (   self)

Definition at line 694 of file dataset.py.

References dataset.Dataset.__getMagneticField(), and dataset.Dataset.__magneticField.

694 
695  def magneticField( self ):
696  if not self.__magneticField:
697  self.__magneticField = self.__getMagneticField()
698  return self.__magneticField
dataset.Dataset.magneticField
def magneticField
Definition: dataset.py:694
dataset.Dataset.__getMagneticField
def __getMagneticField
Definition: dataset.py:406
def dataset.Dataset.magneticFieldForRun (   self,
  run = -1 
)

Definition at line 699 of file dataset.py.

References dataset.Dataset.__getMagneticFieldForRun().

699 
700  def magneticFieldForRun( self, run = -1 ):
701  return self.__getMagneticFieldForRun(run)
dataset.Dataset.__getMagneticFieldForRun
def __getMagneticFieldForRun
Definition: dataset.py:477
dataset.Dataset.magneticFieldForRun
def magneticFieldForRun
Definition: dataset.py:699
def dataset.Dataset.name (   self)

Definition at line 843 of file dataset.py.

References dataset.Dataset.__name.

Referenced by cuy.divideElement.__init__(), cuy.plotElement.__init__(), cuy.additionElement.__init__(), cuy.superimposeElement.__init__(), cuy.graphElement.__init__(), config.CFG.__str__(), validation.Sample.digest(), VIDSelectorBase.VIDSelectorBase.initialize(), and Vispa.Views.PropertyView.Property.valueChanged().

843 
844  def name( self ):
845  return self.__name
def dataset.Dataset.parentDataset (   self)

Definition at line 702 of file dataset.py.

References dataset.Dataset.__getParentDataset(), dataset.Dataset.__parentDataset, and dataset.Dataset.datasetSnippet().

Referenced by dataset.Dataset.__getFileInfoList().

702 
703  def parentDataset( self ):
704  if not self.__parentDataset:
705  self.__parentDataset = self.__getParentDataset()
706  return self.__parentDataset
dataset.Dataset.parentDataset
def parentDataset
Definition: dataset.py:702
dataset.Dataset.__getParentDataset
def __getParentDataset
Definition: dataset.py:396
def dataset.Dataset.predefined (   self)

Definition at line 846 of file dataset.py.

References dataset.Dataset.__predefined.

Referenced by dataset.Dataset.__createSnippet().

846 
847  def predefined( self ):
848  return self.__predefined
def dataset.Dataset.printInfo (   self)

Definition at line 321 of file dataset.py.

References dataset.EOSDataset.castorDir, dataset.Dataset.castorDir, dataset.Dataset.lfnDir, ElectronMVAID.ElectronMVAID.name, counter.Counter.name, average.Average.name, entry.name, geometrydata.GeometryData.name, histograms.Histograms.name, TmModule.name, cond::persistency::TAG::NAME.name, cond::persistency::GLOBAL_TAG::NAME.name, core.autovars.NTupleVariable.name, cond::persistency::TAG::TIME_TYPE.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::TAG::OBJECT_TYPE.name, genericValidation.GenericValidation.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, cond::persistency::TAG::SYNCHRONIZATION.name, preexistingValidation.PreexistingValidation.name, cond::persistency::GLOBAL_TAG::RELEASE.name, ora::RecordSpecImpl::Item.name, MEPSet.name, cond::persistency::TAG::END_OF_VALIDITY.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, cond::persistency::TAG::DESCRIPTION.name, cond::persistency::GTEditorData.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, FWTGeoRecoGeometry::Info.name, Types._Untracked.name, cond::persistency::TAG::INSERTION_TIME.name, cond::persistency::TAG::MODIFICATION_TIME.name, dataset.BaseDataset.name, OutputMEPSet.name, personalPlayback.Applet.name, ParameterSet.name, PixelDCSObject< class >::Item.name, analyzer.Analyzer.name, DQMRivetClient::LumiOption.name, MagCylinder.name, alignment.Alignment.name, ParSet.name, DQMRivetClient::ScaleFactorOption.name, SingleObjectCondition.name, EgHLTOfflineSummaryClient::SumHistBinData.name, XMLHTRZeroSuppressionLoader::_loaderBaseConfig.name, XMLRBXPedestalsLoader::_loaderBaseConfig.name, DQMGenericClient::EfficOption.name, cond::persistency::GTProxyData.name, core.autovars.NTupleObjectType.name, o2o.O2OJob.name, MyWatcher.name, edm::PathTimingSummary.name, lumi::TriggerInfo.name, cond::TimeTypeSpecs.name, edm::PathSummary.name, PixelEndcapLinkMaker::Item.name, perftools::EdmEventSize::BranchRecord.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, FWTableViewManager::TableEntry.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, PixelBarrelLinkMaker::Item.name, Mapper::definition< ScannerT >.name, EcalLogicID.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, ExpressionHisto< T >.name, XMLProcessor::_loaderBaseConfig.name, DQMGenericClient::ProfileOption.name, TreeCrawler.Package.name, cond::persistency::PAYLOAD::HASH.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, cond::persistency::PAYLOAD::DATA.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, cond::persistency::PAYLOAD::VERSION.name, options.ConnectionHLTMenu.name, MagGeoBuilderFromDDD::volumeHandle.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, DQMGenericClient::NormOption.name, Node.name, DQMGenericClient::CDOption.name, FastHFShowerLibrary.name, h4DSegm.name, PhysicsTools::Calibration::Variable.name, cond::TagInfo_t.name, CounterChecker.name, EDMtoMEConverter.name, looper.Looper.name, MEtoEDM< T >::MEtoEDMObject.name, cond::persistency::IOV::TAG_NAME.name, TrackerSectorStruct.name, cond::persistency::IOV::SINCE.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, classes.MonitorData.name, MuonGeometrySanityCheckPoint.name, classes.OutputData.name, options.HLTProcessOptions.name, h2DSegm.name, config.Analyzer.name, core.autovars.NTupleSubObject.name, DQMNet::WaitObject.name, AlpgenParameterName.name, SiStripMonitorDigi.name, core.autovars.NTupleObject.name, geometry.Structure.name, cond::persistency::TAG_LOG::TAG_NAME.name, cond::persistency::TAG_LOG::EVENT_TIME.name, cond::persistency::TAG_LOG::USER_NAME.name, cond::persistency::TAG_LOG::HOST_NAME.name, cond::persistency::TAG_LOG::COMMAND.name, cond::persistency::TAG_LOG::ACTION.name, cond::persistency::TAG_LOG::USER_TEXT.name, config.Service.name, core.autovars.NTupleCollection.name, FastTimerService::LuminosityDescription.name, personalPlayback.FrameworkJob.name, plotscripts.SawTeethFunction.name, FastTimerService::ProcessDescription.name, hTMaxCell.name, cscdqm::ParHistoDef.name, BeautifulSoup.Tag.name, SummaryOutputProducer::GenericSummary.name, TiXmlAttribute.name, and BeautifulSoup.SoupStrainer.name.

321 
322  def printInfo(self):
323  print 'sample : ' + self.name
324  print 'LFN : ' + self.lfnDir
325  print 'Castor path : ' + self.castorDir
def dataset.Dataset.runList (   self)

Definition at line 849 of file dataset.py.

References dataset.Dataset.__getRunList(), and dataset.Dataset.__runList.

849 
850  def runList( self ):
851  if self.__runList:
852  return self.__runList
853  return self.__getRunList()
854 
dataset.Dataset.__getRunList
def __getRunList
Definition: dataset.py:597

Member Data Documentation

dataset.Dataset.__alreadyStored
private

Definition at line 23 of file dataset.py.

Referenced by dataset.Dataset.dump_cff().

dataset.Dataset.__cmssw
private

Definition at line 24 of file dataset.py.

Referenced by dataset.Dataset.__getMagneticField(), and dataset.Dataset.dump_cff().

dataset.Dataset.__cmsswrelease
private

Definition at line 25 of file dataset.py.

Referenced by dataset.Dataset.__getMagneticField().

dataset.Dataset.__dasLimit
private

Definition at line 19 of file dataset.py.

Referenced by dataset.Dataset.fileInfoList().

dataset.Dataset.__dataType
private

Definition at line 76 of file dataset.py.

Referenced by dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.dataType(), and dataset.Dataset.dump_cff().

tuple dataset.Dataset.__dummy_source_template
staticprivate
Initial value:
1 = ("readFiles = cms.untracked.vstring()\n"
2  "secFiles = cms.untracked.vstring()\n"
3  "%(process)ssource = cms.Source(\"PoolSource\",\n"
4  "%(tab)s secondaryFileNames ="
5  "secFiles,\n"
6  "%(tab)s fileNames = readFiles\n"
7  ")\n"
8  "readFiles.extend(['dummy_File.root'])\n"
9  "%(process)smaxEvents = cms.untracked.PSet( "
10  "input = cms.untracked.int32(%(nEvents)s) )\n"
11  "%(skipEventsString)s\n")

Definition at line 103 of file dataset.py.

Referenced by dataset.Dataset.__createSnippet().

dataset.Dataset.__fileInfoList
private

Definition at line 21 of file dataset.py.

Referenced by dataset.Dataset.__getFileInfoList().

dataset.Dataset.__fileList
private

Definition at line 20 of file dataset.py.

Referenced by dataset.Dataset.fileList().

dataset.Dataset.__filename
private

Definition at line 53 of file dataset.py.

Referenced by dataset.Dataset.__getDataType(), dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.datasetSnippet(), csvReporter.csvReporter.writeRow(), and csvReporter.csvReporter.writeRows().

dataset.Dataset.__firstusedrun
private

Definition at line 26 of file dataset.py.

Referenced by dataset.Dataset.__createSnippet(), dataset.Dataset.__getMagneticFieldForRun(), and dataset.Dataset.forcerunrange().

dataset.Dataset.__lastusedrun
private

Definition at line 27 of file dataset.py.

Referenced by dataset.Dataset.__createSnippet(), dataset.Dataset.__getMagneticFieldForRun(), and dataset.Dataset.forcerunrange().

dataset.Dataset.__magneticField
private

Definition at line 77 of file dataset.py.

Referenced by dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.dump_cff(), and dataset.Dataset.magneticField().

dataset.Dataset.__name
private

Definition at line 17 of file dataset.py.

Referenced by dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.__getParentDataset(), dataset.Dataset.__getRunList(), dataset.Dataset.convertTimeToRun(), dataset.Dataset.datasetSnippet(), dataset.Dataset.dump_cff(), Config.Process.dumpConfig(), Config.Process.dumpPython(), dataset.Dataset.name(), and Config.Process.name_().

dataset.Dataset.__official
private

Definition at line 34 of file dataset.py.

Referenced by dataset.Dataset.datasetSnippet().

dataset.Dataset.__origName
private

Definition at line 18 of file dataset.py.

Referenced by dataset.Dataset.datasetSnippet().

dataset.Dataset.__parentDataset
private

Definition at line 28 of file dataset.py.

Referenced by dataset.Dataset.parentDataset().

dataset.Dataset.__parentFileInfoList
private

Definition at line 30 of file dataset.py.

Referenced by dataset.Dataset.__getFileInfoList().

dataset.Dataset.__parentFileList
private

Definition at line 29 of file dataset.py.

Referenced by dataset.Dataset.fileList().

dataset.Dataset.__predefined
private

Definition at line 50 of file dataset.py.

Referenced by dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dataset.Dataset.__getMagneticField(), dataset.Dataset.__getMagneticFieldForRun(), dataset.Dataset.datasetSnippet(), and dataset.Dataset.predefined().

dataset.Dataset.__runList
private

Definition at line 22 of file dataset.py.

Referenced by dataset.Dataset.__getRunList(), and dataset.Dataset.runList().

dataset.Dataset.bad_files

Definition at line 282 of file dataset.py.

dataset.Dataset.castorDir

Definition at line 266 of file dataset.py.

Referenced by dataset.Dataset.extractFileSizes(), and dataset.Dataset.printInfo().

tuple dataset.Dataset.dasData = das_client.get_data(dasQuery, dasLimit)
static

Definition at line 340 of file dataset.py.

tuple dataset.Dataset.error = self.__findInJson(jsondict,["data","error"])
static

Definition at line 347 of file dataset.py.

Referenced by argparse.ArgumentParser._get_option_tuples(), python.rootplot.argparse.ArgumentParser._get_option_tuples(), argparse.ArgumentParser._parse_known_args(), python.rootplot.argparse.ArgumentParser._parse_known_args(), argparse.ArgumentParser._parse_optional(), python.rootplot.argparse.ArgumentParser._parse_optional(), argparse.ArgumentParser._read_args_from_files(), python.rootplot.argparse.ArgumentParser._read_args_from_files(), argparse.ArgumentParser.add_subparsers(), python.rootplot.argparse.ArgumentParser.add_subparsers(), argparse.ArgumentParser.parse_args(), python.rootplot.argparse.ArgumentParser.parse_args(), argparse.ArgumentParser.parse_known_args(), and python.rootplot.argparse.ArgumentParser.parse_known_args().

dataset.Dataset.error = None
static

Definition at line 349 of file dataset.py.

Referenced by argparse.ArgumentParser._get_option_tuples(), python.rootplot.argparse.ArgumentParser._get_option_tuples(), argparse.ArgumentParser._parse_known_args(), python.rootplot.argparse.ArgumentParser._parse_known_args(), argparse.ArgumentParser._parse_optional(), python.rootplot.argparse.ArgumentParser._parse_optional(), argparse.ArgumentParser._read_args_from_files(), python.rootplot.argparse.ArgumentParser._read_args_from_files(), argparse.ArgumentParser.add_subparsers(), python.rootplot.argparse.ArgumentParser.add_subparsers(), argparse.ArgumentParser.parse_args(), python.rootplot.argparse.ArgumentParser.parse_args(), argparse.ArgumentParser.parse_known_args(), and python.rootplot.argparse.ArgumentParser.parse_known_args().

dataset.Dataset.files

Definition at line 273 of file dataset.py.

dataset.Dataset.filesAndSizes

Definition at line 311 of file dataset.py.

dataset.Dataset.good_files

Definition at line 283 of file dataset.py.

int dataset.Dataset.i = 0
static

Definition at line 357 of file dataset.py.

tuple dataset.Dataset.jsondict = json.loads( dasData )
static

Definition at line 342 of file dataset.py.

dataset.Dataset.jsondict = dasData
static

Definition at line 344 of file dataset.py.

string dataset.Dataset.jsonfile = "das_query_output_%i.txt"
static

Definition at line 356 of file dataset.py.

dataset.Dataset.jsonfile = jsonfile%i
static

Definition at line 360 of file dataset.py.

tuple dataset.Dataset.jsonstr = self.__findInJson(jsondict,"reason")
static

Definition at line 352 of file dataset.py.

dataset.Dataset.lfnDir

Definition at line 265 of file dataset.py.

Referenced by dataset.Dataset.printInfo().

dataset.Dataset.maskExists

Definition at line 267 of file dataset.py.

string dataset.Dataset.msg = "The DAS query returned an error. The output is very long, and has been stored in:\n"
static

Definition at line 364 of file dataset.py.

Referenced by MatrixReader.MatrixException.__str__(), cmsHarvester.Usage.__str__(), and cmsHarvester.Error.__str__().

dataset.Dataset.report

Definition at line 268 of file dataset.py.

Referenced by addOnTests.testit.run().

tuple dataset.Dataset.theFile = open( jsonfile, "w" )
static

Definition at line 361 of file dataset.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5