dataset.BaseDataset Class Reference

Inheritance diagram for dataset.BaseDataset:

Public Member Functions
def	__init__ (self, name, user, pattern='.*root', run_range=None, dbsInstance=None)
	def init(self, name, user, pattern='. More...
def	buildListOfBadFiles (self)
def	buildListOfFiles (self, pattern)
def	extractFileSizes (self)
def	getPrimaryDatasetEntries (self)
def	listOfFiles (self)
def	listOfGoodFiles (self)
def	listOfGoodFilesWithPrescale (self, prescale)
def	printFiles (self, abspath=True, info=True)
def	printInfo (self)

Public Attributes
	bad_files
	dbsInstance
	MM. More...
	files
	filesAndSizes
	good_files
	name
	pattern
	primaryDatasetEntries
	MM. More...
	report
	run_range
	user

Detailed Description

Definition at line 22 of file dataset.py.

Constructor & Destructor Documentation

__init__()

def dataset.BaseDataset.__init__	(		self,
			name,
			user,
			pattern = '.*root',
			run_range = None,
			dbsInstance = None
	)

def init(self, name, user, pattern='.

*root', run_range=None):

Definition at line 25 of file dataset.py.

    def __init__(self, name, user, pattern='.*root', run_range=None, dbsInstance=None):
        self.name = name
        self.user = user
        self.pattern = pattern
        self.run_range = run_range
        
        self.dbsInstance = dbsInstance
        
        self.primaryDatasetEntries = -1
        self.report = None
        self.buildListOfFiles( self.pattern )
        self.extractFileSizes()
        self.buildListOfBadFiles()
        self.primaryDatasetEntries = self.getPrimaryDatasetEntries()
     

Member Function Documentation

buildListOfBadFiles()

def dataset.BaseDataset.buildListOfBadFiles

(

self

)

Definition at line 48 of file dataset.py.

    def buildListOfBadFiles(self):
        self.good_files = []
        self.bad_files = {}

buildListOfFiles()

def dataset.BaseDataset.buildListOfFiles	(		self,
			pattern
	)

Definition at line 40 of file dataset.py.

Referenced by dataset.BaseDataset.printFiles().

    def buildListOfFiles( self, pattern ):
        self.files = []

extractFileSizes()

def dataset.BaseDataset.extractFileSizes

(

self

)

Get the file size for each file, 
from the eos ls -l command.

Definition at line 43 of file dataset.py.

    def extractFileSizes(self):
        '''Get the file size for each file, 
        from the eos ls -l command.'''
        self.filesAndSizes = {}

getPrimaryDatasetEntries()

def dataset.BaseDataset.getPrimaryDatasetEntries

(

self

)

Definition at line 56 of file dataset.py.

References dataset.BaseDataset.primaryDatasetEntries.

    def getPrimaryDatasetEntries(self):
        return self.primaryDatasetEntries

listOfFiles()

def dataset.BaseDataset.listOfFiles

(

self

)

Returns all files, even the bad ones.

Definition at line 82 of file dataset.py.

References readConfig.fileINI.files, dataset.BaseDataset.files, chain.Chain.files, MatrixReader.MatrixReader.files, MessageLogger.files, MatrixUtil.InputInfo.files, and JsonOutputProducer::JsonConfigurationBlock.files.

    def listOfFiles(self):
        '''Returns all files, even the bad ones.'''
        return self.files

listOfGoodFiles()

def dataset.BaseDataset.listOfGoodFiles

(

self

)

Returns all files flagged as good in the integrity 
check text output, or not present in this file, are 
considered as good.

Definition at line 86 of file dataset.py.

References mps_setup.append, dataset.BaseDataset.bad_files, readConfig.fileINI.files, dataset.BaseDataset.files, chain.Chain.files, MatrixReader.MatrixReader.files, MessageLogger.files, MatrixUtil.InputInfo.files, JsonOutputProducer::JsonConfigurationBlock.files, and dataset.BaseDataset.good_files.

Referenced by dataset.BaseDataset.listOfGoodFilesWithPrescale().

    def listOfGoodFiles(self):
        '''Returns all files flagged as good in the integrity 
        check text output, or not present in this file, are 
        considered as good.'''
        self.good_files = []
        for file in self.files:            
            if file not in self.bad_files:
                self.good_files.append( file )
        return self.good_files

listOfGoodFilesWithPrescale()

def dataset.BaseDataset.listOfGoodFilesWithPrescale	(		self,
			prescale
	)

Takes the list of good files and selects a random sample 
from them according to the prescale factor. 
E.g. a prescale of 10 will select 1 in 10 files.

Definition at line 96 of file dataset.py.

References dataset.BaseDataset.good_files, dataset.int, and dataset.BaseDataset.listOfGoodFiles().

    def listOfGoodFilesWithPrescale(self, prescale):
        """Takes the list of good files and selects a random sample 
        from them according to the prescale factor. 
        E.g. a prescale of 10 will select 1 in 10 files."""

        good_files = self.listOfGoodFiles()
        if prescale < 2:
            return self.good_files
        
        #the number of files to select from the dataset
        num_files = int( (len(good_files)/(1.0*prescale)) + 0.5)
        if num_files < 1:
            num_files = 1
        if num_files > len(good_files):
            num_files = len(good_files)
        
        #pick unique good files randomly
        import random
        subset = set()
        while len(subset) < num_files:
            #pick a random file from the list
            choice = random.choice(good_files)
            slen = len(subset)
            #add to the set
            subset.add(choice)
            #if this was a unique file remove so we don't get 
            #very slow corner cases where prescale is small
            if len(subset) > slen:
                good_files.remove(choice)
        assert len(subset)==num_files,'The number of files does not match'

        return [f for f in subset]

printFiles()

def dataset.BaseDataset.printFiles	(		self,
			abspath = True,
			info = True
	)

Definition at line 59 of file dataset.py.

References dataset.BaseDataset.bad_files, dataset.BaseDataset.buildListOfFiles(), readConfig.fileINI.files, dataset.BaseDataset.files, chain.Chain.files, MatrixReader.MatrixReader.files, MessageLogger.files, MatrixUtil.InputInfo.files, JsonOutputProducer::JsonConfigurationBlock.files, dataset.BaseDataset.filesAndSizes, dataset.BaseDataset.good_files, dataset.BaseDataset.pattern, CSCALCT.pattern, presentation.PageLayout.pattern, l1t::EMTFRoad.pattern, L1TMuon::TriggerPrimitive::CSCData.pattern, cscdqm::MOFilterItem.pattern, CSCCorrelatedLCTDigi.pattern, l1t::EMTFHit.pattern, dataset.BaseDataset.primaryDatasetEntries, and print().

    def printFiles(self, abspath=True, info=True):
        # import pdb; pdb.set_trace()
        if self.files == None:
            self.buildListOfFiles(self.pattern)
        for file in self.files:
            status = 'OK'
            if file in self.bad_files:
                status = self.bad_files[file]
            elif file not in self.good_files:
                status = 'UNKNOWN'
            fileNameToPrint = file
            if abspath == False:
                fileNameToPrint = os.path.basename(file)
            if info:
                size=self.filesAndSizes.get(file,'UNKNOWN').rjust(10)
                # if size is not None:
                #     size = size.rjust(10)
                print(status.ljust(10), size, \
                      '\t', fileNameToPrint)
            else:
                print(fileNameToPrint)
        print('PrimaryDatasetEntries: %d' % self.primaryDatasetEntries)
                

printInfo()

def dataset.BaseDataset.printInfo

(

self

)

Definition at line 52 of file dataset.py.

References ElectronMVAID.ElectronMVAID.name, HcalOfflineHarvesting.name, HcalOnlineHarvesting.name, HFRaddamTask.name, LaserTask.name, NoCQTask.name, PedestalTask.name, QIE10Task.name, QIE11Task.name, RecHitTask.name, UMNioTask.name, ZDCTask.name, AlignableObjectId::entry.name, RawTask.name, average.Average.name, counter.Counter.name, TPTask.name, DigiTask.name, histograms.Histograms.name, LEDTask.name, cond::persistency::TAG::NAME.name, cond::persistency::RUN_INFO::RUN_NUMBER.name, TmModule.name, cond::persistency::GTEditorData.name, cond::persistency::GLOBAL_TAG::NAME.name, cond::persistency::TAG::TIME_TYPE.name, cond::persistency::RUN_INFO::START_TIME.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::RUN_INFO::END_TIME.name, cond::persistency::TAG::OBJECT_TYPE.name, core.autovars.NTupleVariable.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, DQMRivetClient::NormOption.name, cond::persistency::TAG::SYNCHRONIZATION.name, cond::persistency::GLOBAL_TAG::RELEASE.name, cond::persistency::TAG::END_OF_VALIDITY.name, MEPSet.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, cond::persistency::O2O_RUN::JOB_NAME.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::TAG::DESCRIPTION.name, cms::dd::NameValuePair< T >.name, cond::persistency::O2O_RUN::START_TIME.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, FWTGeoRecoGeometry::Info.name, cond::persistency::O2O_RUN::END_TIME.name, cond::persistency::TAG::INSERTION_TIME.name, cond::persistency::O2O_RUN::STATUS_CODE.name, cond::persistency::TAG::MODIFICATION_TIME.name, cond::persistency::O2O_RUN::LOG.name, ParameterSet.name, nanoaod::MergeableCounterTable::SingleColumn< T >.name, cond::persistency::TAG::PROTECTION_CODE.name, preexistingValidation.PreexistingValidation.name, OutputMEPSet.name, MEPSetData.name, AlignmentConstraint.name, PixelDCSObject< T >::Item.name, dataset.BaseDataset.name, cms::dd::ValuePair< T, U >.name, personalPlayback.Applet.name, Types._Untracked.name, analyzer.Analyzer.name, MagCylinder.name, DQMRivetClient::LumiOption.name, heppy::ParSet.name, cond::persistency::GTProxyData.name, SingleObjectCondition.name, edm::PathTimingSummary.name, DQMRivetClient::ScaleFactorOption.name, EgHLTOfflineSummaryClient::SumHistBinData.name, cms::DDAlgoArguments.name, Barrel.name, cond::TimeTypeSpecs.name, perftools::EdmEventSize::BranchRecord.name, core.autovars.NTupleObjectType.name, edm::PathSummary.name, EcalLogicID.name, alignment.Alignment.name, lumi::TriggerInfo.name, PixelEndcapLinkMaker::Item.name, XMLProcessor::_loaderBaseConfig.name, FWTableViewManager::TableEntry.name, MEtoEDM< T >::MEtoEDMObject.name, PixelBarrelLinkMaker::Item.name, ExpressionHisto< T >.name, DQMGenericClient::EfficOption.name, TreeCrawler.Package.name, Supermodule.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, genericValidation.GenericValidation.name, options.ConnectionHLTMenu.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, cms::DDParsingContext::CompositeMaterial.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, cond::Tag_t.name, dqmoffline::l1t::HistDefinition.name, DQMGenericClient::ProfileOption.name, nanoaod::MergeableCounterTable::VectorColumn< T >.name, magneticfield::BaseVolumeHandle.name, FastHFShowerLibrary.name, emtf::Node.name, h4DSegm.name, DQMGenericClient::NormOption.name, core.TriggerMatchAnalyzer.TriggerMatchAnalyzer.name, DQMGenericClient::CDOption.name, CounterChecker.name, PhysicsTools::Calibration::Variable.name, cond::TagInfo_t.name, TrackerSectorStruct.name, MuonGeometrySanityCheckPoint.name, FCDTask.name, cond::persistency::PAYLOAD::HASH.name, DQMGenericClient::NoFlowOption.name, looper.Looper.name, Mapper::definition< ScannerT >.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, cond::persistency::PAYLOAD::DATA.name, EDMtoMEConverter.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, cond::persistency::PAYLOAD::VERSION.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, classes.MonitorData.name, HistogramManager.name, classes.OutputData.name, BPHDecayToResResBuilderBase::DZSelect.name, Crystal.name, h2DSegm.name, options.HLTProcessOptions.name, cond::persistency::IOV::TAG_NAME.name, cond::persistency::IOV::SINCE.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, DQMNet::WaitObject.name, core.TriggerBitAnalyzer.TriggerBitAnalyzer.name, AlpgenParameterName.name, config.Analyzer.name, geometry.Structure.name, core.autovars.NTupleSubObject.name, Capsule.name, core.autovars.NTupleObject.name, Ceramic.name, SiStripMonitorDigi.name, config.Service.name, BulkSilicon.name, APD.name, core.autovars.NTupleCollection.name, nanoaod::FlatTable::Column.name, BPHRecoBuilder::BPHRecoSource.name, BPHRecoBuilder::BPHCompSource.name, StraightTrackAlignment::RPSetPlots.name, cond::persistency::TAG_AUTHORIZATION::TAG_NAME.name, cond::persistency::TAG_AUTHORIZATION::ACCESS_TYPE.name, cond::persistency::TAG_AUTHORIZATION::CREDENTIAL.name, cond::persistency::TAG_AUTHORIZATION::CREDENTIAL_TYPE.name, InnerLayerVolume.name, cond::payloadInspector::TagReference.name, cond::persistency::TAG_LOG::TAG_NAME.name, cond::persistency::TAG_LOG::EVENT_TIME.name, cond::persistency::TAG_LOG::USER_NAME.name, cond::persistency::TAG_LOG::HOST_NAME.name, cond::persistency::TAG_LOG::COMMAND.name, cond::persistency::TAG_LOG::ACTION.name, cond::persistency::TAG_LOG::USER_TEXT.name, personalPlayback.FrameworkJob.name, Grid.name, Grille.name, BackPipe.name, plotscripts.SawTeethFunction.name, PatchPanel.name, BackCoolTank.name, DryAirTube.name, crabFunctions.CrabTask.name, MBCoolTube.name, MBManif.name, cscdqm::ParHistoDef.name, hTMaxCell.name, BeautifulSoup.Tag.name, SummaryOutputProducer::GenericSummary.name, BeautifulSoup.SoupStrainer.name, print(), dataset.BaseDataset.user, EcalTPGParamReaderFromDB.user, popcon::RpcDataT.user, popcon::RPCObPVSSmapData.user, popcon::RpcObGasData.user, popcon::RpcDataV.user, popcon::RpcDataI.user, popcon::RpcDataGasMix.user, popcon::RpcDataS.user, popcon::RpcDataUXC.user, popcon::RpcDataFebmap.user, EcalDBConnection.user, and MatrixInjector.MatrixInjector.user.

    def printInfo(self):
        print('sample      :  ' + self.name)
        print('user        :  ' + self.user)

Member Data Documentation

bad_files

dataset.BaseDataset.bad_files

Definition at line 50 of file dataset.py.

Referenced by dataset.BaseDataset.listOfGoodFiles(), and dataset.BaseDataset.printFiles().

dbsInstance

dataset.BaseDataset.dbsInstance

MM.

Definition at line 31 of file dataset.py.

Referenced by dataset.PrivateDataset.getPrimaryDatasetEntries().

files

dataset.BaseDataset.files

Definition at line 41 of file dataset.py.

Referenced by dataset.BaseDataset.listOfFiles(), dataset.BaseDataset.listOfGoodFiles(), and dataset.BaseDataset.printFiles().

filesAndSizes

dataset.BaseDataset.filesAndSizes

Definition at line 46 of file dataset.py.

Referenced by dataset.BaseDataset.printFiles().

good_files

dataset.BaseDataset.good_files

Definition at line 49 of file dataset.py.

Referenced by dataset.BaseDataset.listOfGoodFiles(), dataset.BaseDataset.listOfGoodFilesWithPrescale(), and dataset.BaseDataset.printFiles().

name

dataset.BaseDataset.name

Definition at line 26 of file dataset.py.

Referenced by FWLite.ElectronMVAID.__call__(), dirstructure.Directory.__create_pie_image(), DisplayManager.DisplayManager.__del__(), dqm_interfaces.DirID.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dirstructure.Comparison.__make_image(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.CFG.__str__(), FWLite.WorkingPoints._reformat_cut_definitions(), dataset.CMSDataset.buildListOfFiles(), dataset.LocalDataset.buildListOfFiles(), dataset.CMSDataset.buildListOfFilesDBS(), dirstructure.Directory.calcStats(), validation.Sample.digest(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), DisplayManager.DisplayManager.Draw(), utils.StatisticalTest.get_status(), production_tasks.Task.getname(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), VIDSelectorBase.VIDSelectorBase.initialize(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), production_tasks.MonitorJobs.run(), python.rootplot.utilities.Hist.TGraph(), and python.rootplot.utilities.Hist.TH1F().

pattern

dataset.BaseDataset.pattern

Definition at line 28 of file dataset.py.

Referenced by dataset.BaseDataset.printFiles().

primaryDatasetEntries

dataset.BaseDataset.primaryDatasetEntries

MM.

Definition at line 33 of file dataset.py.

Referenced by dataset.BaseDataset.getPrimaryDatasetEntries(), and dataset.BaseDataset.printFiles().

report

dataset.BaseDataset.report

Definition at line 34 of file dataset.py.

Referenced by dataset.Dataset.getPrimaryDatasetEntries(), and addOnTests.testit.run().

run_range

dataset.BaseDataset.run_range

Definition at line 29 of file dataset.py.

Referenced by dataset.CMSDataset.buildListOfFiles(), dataset.CMSDataset.buildListOfFilesDBS(), dataset.CMSDataset.getPrimaryDatasetEntries(), and dataset.PrivateDataset.getPrimaryDatasetEntries().

user

dataset.BaseDataset.user

Definition at line 27 of file dataset.py.

Referenced by dataset.BaseDataset.printInfo(), DTWorkflow.DTWorkflow.remote_out_path(), production_tasks.CheckDatasetExists.run(), production_tasks.GenerateMask.run(), production_tasks.SourceCFG.run(), production_tasks.FullCFG.run(), production_tasks.MonitorJobs.run(), and production_tasks.CleanJobFiles.run().