CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
List of all members | Public Member Functions | Public Attributes
dataset.BaseDataset Class Reference
Inheritance diagram for dataset.BaseDataset:
dataset.CMSDataset dataset.Dataset dataset.EOSDataset dataset.LocalDataset dataset.PrivateDataset

Public Member Functions

def __init__
 def init(self, name, user, pattern='. More...
 
def buildListOfBadFiles
 
def buildListOfFiles
 
def extractFileSizes
 
def getPrimaryDatasetEntries
 
def listOfFiles
 
def listOfGoodFiles
 
def listOfGoodFilesWithPrescale
 
def printFiles
 
def printInfo
 

Public Attributes

 bad_files
 
 dbsInstance
 MM. More...
 
 files
 
 filesAndSizes
 
 good_files
 
 name
 
 pattern
 
 primaryDatasetEntries
 MM. More...
 
 report
 
 run_range
 
 user
 

Detailed Description

Definition at line 22 of file dataset.py.

Constructor & Destructor Documentation

def dataset.BaseDataset.__init__ (   self,
  name,
  user,
  pattern = '.*root',
  run_range = None,
  dbsInstance = None 
)

def init(self, name, user, pattern='.

*root', run_range=None):

Definition at line 25 of file dataset.py.

25 
26  def __init__(self, name, user, pattern='.*root', run_range=None, dbsInstance=None):
27  self.name = name
28  self.user = user
29  self.pattern = pattern
30  self.run_range = run_range
31  ### MM
32  self.dbsInstance = dbsInstance
33  ### MM
34  self.primaryDatasetEntries = -1
35  self.report = None
36  self.buildListOfFiles( self.pattern )
37  self.extractFileSizes()
38  self.buildListOfBadFiles()
39  self.primaryDatasetEntries = self.getPrimaryDatasetEntries()
dataset.BaseDataset.getPrimaryDatasetEntries
def getPrimaryDatasetEntries
Definition: dataset.py:56
dataset.BaseDataset.__init__
def __init__
def init(self, name, user, pattern='.
Definition: dataset.py:25
dataset.BaseDataset.primaryDatasetEntries
primaryDatasetEntries
MM.
Definition: dataset.py:33
dataset.BaseDataset.buildListOfBadFiles
def buildListOfBadFiles
Definition: dataset.py:48

Member Function Documentation

def dataset.BaseDataset.buildListOfBadFiles (   self)

Definition at line 48 of file dataset.py.

48 
49  def buildListOfBadFiles(self):
50  self.good_files = []
51  self.bad_files = {}
dataset.BaseDataset.buildListOfBadFiles
def buildListOfBadFiles
Definition: dataset.py:48
def dataset.BaseDataset.buildListOfFiles (   self,
  pattern 
)

Definition at line 40 of file dataset.py.

Referenced by dataset.BaseDataset.printFiles().

40 
41  def buildListOfFiles( self, pattern ):
42  self.files = []
def dataset.BaseDataset.extractFileSizes (   self) 
Get the file size for each file, 
from the eos ls -l command.

Definition at line 43 of file dataset.py.

43 
44  def extractFileSizes(self):
45  '''Get the file size for each file,
46  from the eos ls -l command.'''
47  self.filesAndSizes = {}
def dataset.BaseDataset.getPrimaryDatasetEntries (   self)

Definition at line 56 of file dataset.py.

References dataset.BaseDataset.primaryDatasetEntries.

56 
57  def getPrimaryDatasetEntries(self):
58  return self.primaryDatasetEntries
dataset.BaseDataset.getPrimaryDatasetEntries
def getPrimaryDatasetEntries
Definition: dataset.py:56
dataset.BaseDataset.primaryDatasetEntries
primaryDatasetEntries
MM.
Definition: dataset.py:33
def dataset.BaseDataset.listOfFiles (   self) 
Returns all files, even the bad ones.

Definition at line 82 of file dataset.py.

References readConfig.fileINI.files, dataset.BaseDataset.files, chain.Chain.files, MatrixReader.MatrixReader.files, MatrixUtil.InputInfo.files, and JsonOutputProducer::JsonConfigurationBlock.files.

82 
83  def listOfFiles(self):
84  '''Returns all files, even the bad ones.'''
85  return self.files
def dataset.BaseDataset.listOfGoodFiles (   self) 
Returns all files flagged as good in the integrity 
check text output, or not present in this file, are 
considered as good.

Definition at line 86 of file dataset.py.

References dataset.BaseDataset.bad_files, readConfig.fileINI.files, dataset.BaseDataset.files, chain.Chain.files, MatrixReader.MatrixReader.files, MatrixUtil.InputInfo.files, JsonOutputProducer::JsonConfigurationBlock.files, and dataset.BaseDataset.good_files.

Referenced by dataset.BaseDataset.listOfGoodFilesWithPrescale().

86 
87  def listOfGoodFiles(self):
88  '''Returns all files flagged as good in the integrity
89  check text output, or not present in this file, are
90  considered as good.'''
91  self.good_files = []
92  for file in self.files:
93  if file not in self.bad_files:
94  self.good_files.append( file )
95  return self.good_files
def dataset.BaseDataset.listOfGoodFilesWithPrescale (   self,
  prescale 
) 
Takes the list of good files and selects a random sample 
from them according to the prescale factor. 
E.g. a prescale of 10 will select 1 in 10 files.

Definition at line 96 of file dataset.py.

References dataset.BaseDataset.good_files, and dataset.BaseDataset.listOfGoodFiles().

96 
97  def listOfGoodFilesWithPrescale(self, prescale):
98  """Takes the list of good files and selects a random sample
99  from them according to the prescale factor.
100  E.g. a prescale of 10 will select 1 in 10 files."""
101 
102  good_files = self.listOfGoodFiles()
103  if prescale < 2:
104  return self.good_files
105 
106  #the number of files to select from the dataset
107  num_files = int( (len(good_files)/(1.0*prescale)) + 0.5)
108  if num_files < 1:
109  num_files = 1
110  if num_files > len(good_files):
111  num_files = len(good_files)
112 
113  #pick unique good files randomly
114  import random
115  subset = set()
116  while len(subset) < num_files:
117  #pick a random file from the list
118  choice = random.choice(good_files)
119  slen = len(subset)
120  #add to the set
121  subset.add(choice)
122  #if this was a unique file remove so we don't get
123  #very slow corner cases where prescale is small
124  if len(subset) > slen:
125  good_files.remove(choice)
126  assert len(subset)==num_files,'The number of files does not match'
127 
128  return [f for f in subset]
dataset.BaseDataset.listOfGoodFilesWithPrescale
def listOfGoodFilesWithPrescale
Definition: dataset.py:96
def dataset.BaseDataset.printFiles (   self,
  abspath = True,
  info = True 
)

Definition at line 59 of file dataset.py.

References dataset.BaseDataset.bad_files, dataset.BaseDataset.buildListOfFiles(), readConfig.fileINI.files, dataset.BaseDataset.files, chain.Chain.files, MatrixReader.MatrixReader.files, MatrixUtil.InputInfo.files, JsonOutputProducer::JsonConfigurationBlock.files, dataset.BaseDataset.good_files, CSCALCT.pattern, dataset.BaseDataset.pattern, presentation.PageLayout.pattern, l1t::EMTFRoad.pattern, L1TMuon::TriggerPrimitive::CSCData.pattern, cscdqm::MOFilterItem.pattern, CSCCorrelatedLCTDigi.pattern, l1t::EMTFHit.pattern, dataset.BaseDataset.primaryDatasetEntries, and print().

59 
60  def printFiles(self, abspath=True, info=True):
61  # import pdb; pdb.set_trace()
62  if self.files == None:
63  self.buildListOfFiles(self.pattern)
64  for file in self.files:
65  status = 'OK'
66  if file in self.bad_files:
67  status = self.bad_files[file]
68  elif file not in self.good_files:
69  status = 'UNKNOWN'
70  fileNameToPrint = file
71  if abspath == False:
72  fileNameToPrint = os.path.basename(file)
73  if info:
74  size=self.filesAndSizes.get(file,'UNKNOWN').rjust(10)
75  # if size is not None:
76  # size = size.rjust(10)
77  print(status.ljust(10), size, \
78  '\t', fileNameToPrint)
79  else:
80  print(fileNameToPrint)
81  print('PrimaryDatasetEntries: %d' % self.primaryDatasetEntries)
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
dataset.BaseDataset.primaryDatasetEntries
primaryDatasetEntries
MM.
Definition: dataset.py:33
def dataset.BaseDataset.printInfo (   self)

Definition at line 52 of file dataset.py.

References ElectronMVAID.ElectronMVAID.name, AlignableObjectId::entry.name, average.Average.name, counter.Counter.name, histograms.Histograms.name, TmModule.name, cond::persistency::TAG::NAME.name, cond::persistency::RUN_INFO::RUN_NUMBER.name, cond::persistency::GTEditorData.name, cond::persistency::GLOBAL_TAG::NAME.name, cond::persistency::TAG::TIME_TYPE.name, cond::persistency::RUN_INFO::START_TIME.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::RUN_INFO::END_TIME.name, core.autovars.NTupleVariable.name, cond::persistency::TAG::OBJECT_TYPE.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, cond::persistency::TAG::SYNCHRONIZATION.name, DQMRivetClient::NormOption.name, cond::persistency::GLOBAL_TAG::RELEASE.name, cond::persistency::TAG::END_OF_VALIDITY.name, MEPSet.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, cond::persistency::O2O_RUN::JOB_NAME.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::TAG::DESCRIPTION.name, cms::dd::NameValuePair< T >.name, cond::persistency::O2O_RUN::START_TIME.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, cond::persistency::O2O_RUN::END_TIME.name, cond::persistency::TAG::INSERTION_TIME.name, FWTGeoRecoGeometry::Info.name, cond::persistency::O2O_RUN::STATUS_CODE.name, cond::persistency::TAG::MODIFICATION_TIME.name, cond::persistency::O2O_RUN::LOG.name, ParameterSet.name, nanoaod::MergeableCounterTable::SingleColumn< T >.name, cond::persistency::TAG::PROTECTION_CODE.name, preexistingValidation.PreexistingValidation.name, OutputMEPSet.name, AlignmentConstraint.name, PixelDCSObject< class >::Item.name, dataset.BaseDataset.name, cms::dd::ValuePair< T, U >.name, personalPlayback.Applet.name, Types._Untracked.name, MagCylinder.name, analyzer.Analyzer.name, heppy::ParSet.name, DQMRivetClient::LumiOption.name, o2olib.O2OJob.name, cond::persistency::GTProxyData.name, SingleObjectCondition.name, EgHLTOfflineSummaryClient::SumHistBinData.name, edm::PathTimingSummary.name, DQMRivetClient::ScaleFactorOption.name, cms::DDAlgoArguments.name, Barrel.name, perftools::EdmEventSize::BranchRecord.name, core.autovars.NTupleObjectType.name, cond::TimeTypeSpecs.name, edm::PathSummary.name, EcalLogicID.name, alignment.Alignment.name, lumi::TriggerInfo.name, PixelEndcapLinkMaker::Item.name, XMLProcessor::_loaderBaseConfig.name, MEtoEDM< T >::MEtoEDMObject.name, FWTableViewManager::TableEntry.name, PixelBarrelLinkMaker::Item.name, ExpressionHisto< T >.name, DQMGenericClient::EfficOption.name, TreeCrawler.Package.name, Supermodule.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, genericValidation.GenericValidation.name, options.ConnectionHLTMenu.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, cms::DDParsingContext::CompositeMaterial.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, cond::Tag_t.name, dqmoffline::l1t::HistDefinition.name, DQMGenericClient::ProfileOption.name, magneticfield::BaseVolumeHandle.name, nanoaod::MergeableCounterTable::VectorColumn< T >.name, FastHFShowerLibrary.name, emtf::Node.name, h4DSegm.name, DQMGenericClient::NormOption.name, DQMGenericClient::CDOption.name, CounterChecker.name, PhysicsTools::Calibration::Variable.name, cond::TagInfo_t.name, TrackerSectorStruct.name, MuonGeometrySanityCheckPoint.name, cond::persistency::PAYLOAD::HASH.name, DQMGenericClient::NoFlowOption.name, Mapper::definition< ScannerT >.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, looper.Looper.name, cond::persistency::PAYLOAD::DATA.name, EDMtoMEConverter.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, cond::persistency::PAYLOAD::VERSION.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, classes.MonitorData.name, HistogramManager.name, classes.OutputData.name, Crystal.name, h2DSegm.name, options.HLTProcessOptions.name, cond::persistency::IOV::TAG_NAME.name, cond::persistency::IOV::SINCE.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, DQMNet::WaitObject.name, AlpgenParameterName.name, config.Analyzer.name, geometry.Structure.name, core.autovars.NTupleSubObject.name, Capsule.name, core.autovars.NTupleObject.name, Ceramic.name, SiStripMonitorDigi.name, BulkSilicon.name, config.Service.name, APD.name, nanoaod::FlatTable::Column.name, core.autovars.NTupleCollection.name, BPHRecoBuilder::BPHRecoSource.name, BPHRecoBuilder::BPHCompSource.name, StraightTrackAlignment::RPSetPlots.name, cond::persistency::TAG_AUTHORIZATION::TAG_NAME.name, cond::persistency::TAG_AUTHORIZATION::ACCESS_TYPE.name, cond::persistency::TAG_AUTHORIZATION::CREDENTIAL.name, cond::persistency::TAG_AUTHORIZATION::CREDENTIAL_TYPE.name, InnerLayerVolume.name, cond::payloadInspector::TagReference.name, cond::persistency::TAG_LOG::TAG_NAME.name, cond::persistency::TAG_LOG::EVENT_TIME.name, cond::persistency::TAG_LOG::USER_NAME.name, cond::persistency::TAG_LOG::HOST_NAME.name, cond::persistency::TAG_LOG::COMMAND.name, cond::persistency::TAG_LOG::ACTION.name, cond::persistency::TAG_LOG::USER_TEXT.name, personalPlayback.FrameworkJob.name, Grid.name, Grille.name, BackPipe.name, plotscripts.SawTeethFunction.name, PatchPanel.name, BackCoolTank.name, DryAirTube.name, crabFunctions.CrabTask.name, MBCoolTube.name, MBManif.name, cscdqm::ParHistoDef.name, hTMaxCell.name, BeautifulSoup.Tag.name, SummaryOutputProducer::GenericSummary.name, BeautifulSoup.SoupStrainer.name, print(), dataset.BaseDataset.user, EcalTPGParamReaderFromDB.user, popcon::RpcObGasData.user, popcon::RpcDataT.user, popcon::RpcDataV.user, popcon::RPCObPVSSmapData.user, popcon::RpcDataI.user, popcon::RpcDataFebmap.user, popcon::RpcDataGasMix.user, popcon::RpcDataS.user, popcon::RpcDataUXC.user, EcalDBConnection.user, and MatrixInjector.MatrixInjector.user.

52 
53  def printInfo(self):
54  print('sample : ' + self.name)
55  print('user : ' + self.user)
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

Member Data Documentation

dataset.BaseDataset.bad_files

Definition at line 50 of file dataset.py.

Referenced by dataset.BaseDataset.listOfGoodFiles(), and dataset.BaseDataset.printFiles().

dataset.BaseDataset.dbsInstance

MM.

Definition at line 31 of file dataset.py.

Referenced by dataset.PrivateDataset.getPrimaryDatasetEntries().

dataset.BaseDataset.files

Definition at line 41 of file dataset.py.

Referenced by dataset.BaseDataset.listOfFiles(), dataset.BaseDataset.listOfGoodFiles(), and dataset.BaseDataset.printFiles().

dataset.BaseDataset.filesAndSizes

Definition at line 46 of file dataset.py.

dataset.BaseDataset.good_files

Definition at line 49 of file dataset.py.

Referenced by dataset.BaseDataset.listOfGoodFiles(), dataset.BaseDataset.listOfGoodFilesWithPrescale(), and dataset.BaseDataset.printFiles().

dataset.BaseDataset.name

Definition at line 26 of file dataset.py.

Referenced by FWLite.ElectronMVAID.__call__(), dirstructure.Directory.__create_pie_image(), DisplayManager.DisplayManager.__del__(), dqm_interfaces.DirID.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dirstructure.Comparison.__make_image(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.CFG.__str__(), FWLite.WorkingPoints._reformat_cut_definitions(), dirstructure.Directory.calcStats(), geometryComparison.GeometryComparison.createScript(), validation.Sample.digest(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), DisplayManager.DisplayManager.Draw(), utils.StatisticalTest.get_status(), production_tasks.Task.getname(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), primaryVertexResolution.PrimaryVertexResolution.getRepMap(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), VIDSelectorBase.VIDSelectorBase.initialize(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), production_tasks.MonitorJobs.run(), python.rootplot.utilities.Hist.TGraph(), and python.rootplot.utilities.Hist.TH1F().

dataset.BaseDataset.pattern

Definition at line 28 of file dataset.py.

Referenced by dataset.BaseDataset.printFiles().

dataset.BaseDataset.primaryDatasetEntries

MM.

Definition at line 33 of file dataset.py.

Referenced by dataset.BaseDataset.getPrimaryDatasetEntries(), and dataset.BaseDataset.printFiles().

dataset.BaseDataset.report

Definition at line 34 of file dataset.py.

Referenced by dataset.Dataset.getPrimaryDatasetEntries(), and addOnTests.testit.run().

dataset.BaseDataset.run_range

Definition at line 29 of file dataset.py.

Referenced by dataset.CMSDataset.buildListOfFiles(), dataset.CMSDataset.buildListOfFilesDBS(), dataset.CMSDataset.getPrimaryDatasetEntries(), and dataset.PrivateDataset.getPrimaryDatasetEntries().

dataset.BaseDataset.user

Definition at line 27 of file dataset.py.

Referenced by dataset.BaseDataset.printInfo(), DTWorkflow.DTWorkflow.remote_out_path(), production_tasks.CheckDatasetExists.run(), production_tasks.GenerateMask.run(), production_tasks.SourceCFG.run(), production_tasks.FullCFG.run(), production_tasks.MonitorJobs.run(), and production_tasks.CleanJobFiles.run().

Generated for CMSSW Reference Manual by doxygen 1.8.5