dataset.CMSDataset Class Reference

Inheritance diagram for dataset.CMSDataset:

Public Member Functions
def	__init__ (self, name, run_range=None)
def	buildListOfFiles (self, pattern='.*root')
def	buildListOfFilesDBS (self, pattern, begin=-1, end=-1)
def	getPrimaryDatasetEntries (self)
Public Member Functions inherited from dataset.BaseDataset
def	__init__ (self, name, user, pattern='.*root', run_range=None, dbsInstance=None)
	def init(self, name, user, pattern='. More...
def	buildListOfBadFiles (self)
def	buildListOfFiles (self, pattern)
def	extractFileSizes (self)
def	getPrimaryDatasetEntries (self)
def	listOfFiles (self)
def	listOfGoodFiles (self)
def	listOfGoodFilesWithPrescale (self, prescale)
def	printFiles (self, abspath=True, info=True)
def	printInfo (self)

Static Public Member Functions
def	findPrimaryDatasetEntries (dataset, runmin, runmax)
def	findPrimaryDatasetNumFiles (dataset, runmin, runmax)

Public Attributes
	files
Public Attributes inherited from dataset.BaseDataset
	bad_files
	dbsInstance
	MM. More...
	files
	filesAndSizes
	good_files
	name
	pattern
	primaryDatasetEntries
	MM. More...
	report
	run_range
	user

Detailed Description

Definition at line 128 of file dataset.py.

Constructor & Destructor Documentation

def dataset.CMSDataset.__init__	(		self,
			name,
			run_range = None
	)

Definition at line 130 of file dataset.py.

    def __init__(self, name, run_range = None):
        super(CMSDataset, self).__init__( name, 'CMS', run_range=run_range)

Member Function Documentation

def dataset.CMSDataset.buildListOfFiles	(		self,
			pattern = '.*root'
	)

Definition at line 162 of file dataset.py.

References dataset.CMSDataset.findPrimaryDatasetNumFiles(), dataset.int, and dataset.BaseDataset.run_range.

    def buildListOfFiles(self, pattern='.*root'):
        runs = (-1,-1)
        if self.run_range is not None:
            runs = self.run_range
        num_files=self.findPrimaryDatasetNumFiles(self.name.rstrip('/'),
                                                  runs[0],runs[1])
        limit = 10000
        if num_files > limit:
            num_steps = int(num_files/limit)+1
            self.files = []
            for i in xrange(num_steps):
                DBSFiles=self.buildListOfFilesDBS(pattern,
                                                  i*limit,
                                                  ((i+1)*limit)-1)
                self.files.extend(DBSFiles)
        else:
            self.files = self.buildListOfFilesDBS(pattern)
            

def dataset.CMSDataset.buildListOfFilesDBS	(		self,
			pattern,
			begin = -1,
			end = -1
	)

Definition at line 133 of file dataset.py.

References edm.print(), and dataset.BaseDataset.run_range.

    def buildListOfFilesDBS(self, pattern, begin=-1, end=-1):
        print('buildListOfFilesDBS',begin,end)
        sampleName = self.name.rstrip('/')
        query, qwhat = sampleName, "dataset"
        if "#" in sampleName: qwhat = "block"
        if self.run_range is not None and self.run_range != (-1,-1):
            if self.run_range[0] == self.run_range[1]:
                query += "   run=%s" % self.run_range[0]
            else:
                print("WARNING: queries with run ranges are slow in DAS")
                query += "   run between [%s,%s]" % ( self.run_range[0],self.run_range[1] )
        dbs='das_client.py --query="file %s=%s"'%(qwhat,query)
        if begin >= 0:
            dbs += ' --index %d' % begin
        if end >= 0:
            dbs += ' --limit %d' % (end-begin+1)
        else:
            dbs += ' --limit 0' 
        print('dbs\t: %s' % dbs)
        dbsOut = os.popen(dbs)
        files = []
        for line in dbsOut:
            if line.find('/store')==-1:
                continue
            line = line.rstrip()
            # print 'line',line
            files.append(line)
        return files

def dataset.CMSDataset.findPrimaryDatasetEntries (   dataset,   runmin,   runmax  )

static

Definition at line 181 of file dataset.py.

References dataset.int, and edm.print().

Referenced by dataset.CMSDataset.getPrimaryDatasetEntries(), and dataset.PrivateDataset.getPrimaryDatasetEntries().

    def findPrimaryDatasetEntries(dataset, runmin, runmax):

        query, qwhat = dataset, "dataset"
        if "#" in dataset: qwhat = "block"
        if runmin >0 or runmax > 0:
            if runmin == runmax:
                query = "%s run=%d" % (query,runmin)
            else:
                print("WARNING: queries with run ranges are slow in DAS")
                query = "%s run between [%d, %d]" % (query,runmin if runmin > 0 else 1, runmax if runmax > 0 else 999999)
        dbs='das_client.py --query="summary %s=%s"'%(qwhat,query)
        dbsOut = os.popen(dbs).readlines()

        entries = []
        for line in dbsOut:
            line = line.replace('\n','')
            if "nevents" in line:
                entries.append(int(line.split(":")[1]))
        if entries:
            return sum(entries)
        return -1

def dataset.CMSDataset.findPrimaryDatasetNumFiles (   dataset,   runmin,   runmax  )

static

Definition at line 204 of file dataset.py.

References dataset.int, and edm.print().

Referenced by dataset.CMSDataset.buildListOfFiles(), and dataset.PrivateDataset.buildListOfFilesDBS().

    def findPrimaryDatasetNumFiles(dataset, runmin, runmax):

        query, qwhat = dataset, "dataset"
        if "#" in dataset: qwhat = "block"
        if runmin >0 or runmax > 0:
            if runmin == runmax:
                query = "%s run=%d" % (query,runmin)
            else:
                print("WARNING: queries with run ranges are slow in DAS")
                query = "%s run between [%d, %d]" % (query,runmin if runmin > 0 else 1, runmax if runmax > 0 else 999999)
        dbs='das_client.py --query="summary %s=%s"'%(qwhat,query)
        dbsOut = os.popen(dbs).readlines()

        entries = []
        for line in dbsOut:
            line = line.replace('\n','')
            if "nfiles" in line:
                entries.append(int(line.split(":")[1]))
        if entries:
            return sum(entries)
        return -1

def dataset.CMSDataset.getPrimaryDatasetEntries

(

self

)

Definition at line 226 of file dataset.py.

References dataset.CMSDataset.findPrimaryDatasetEntries(), ElectronMVAID.ElectronMVAID.name, counter.Counter.name, average.Average.name, AlignableObjectId::entry.name, histograms.Histograms.name, cond::persistency::TAG::NAME.name, TmModule.name, cond::persistency::GLOBAL_TAG::NAME.name, core.autovars.NTupleVariable.name, cond::persistency::RUN_INFO::RUN_NUMBER.name, cond::persistency::TAG::TIME_TYPE.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::RUN_INFO::START_TIME.name, cond::persistency::RUN_INFO::END_TIME.name, cond::persistency::TAG::OBJECT_TYPE.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, cond::persistency::TAG::SYNCHRONIZATION.name, cond::persistency::GLOBAL_TAG::RELEASE.name, MEPSet.name, cond::persistency::TAG::END_OF_VALIDITY.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, cond::persistency::TAG::DESCRIPTION.name, cond::persistency::GTEditorData.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, nanoaod::MergeableCounterTable::SingleColumn< T >.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, cond::persistency::TAG::INSERTION_TIME.name, cond::persistency::TAG::MODIFICATION_TIME.name, FWTGeoRecoGeometry::Info.name, Types._Untracked.name, preexistingValidation.PreexistingValidation.name, dataset.BaseDataset.name, OutputMEPSet.name, personalPlayback.Applet.name, ParameterSet.name, PixelDCSObject< T >::Item.name, DQMRivetClient::LumiOption.name, MagCylinder.name, analyzer.Analyzer.name, ParSet.name, DQMRivetClient::ScaleFactorOption.name, EgHLTOfflineSummaryClient::SumHistBinData.name, SingleObjectCondition.name, cond::persistency::GTProxyData.name, core.autovars.NTupleObjectType.name, MyWatcher.name, edm::PathTimingSummary.name, nanoaod::MergeableCounterTable::VectorColumn< T >.name, cms::DDAlgoArguments.name, cond::TimeTypeSpecs.name, lumi::TriggerInfo.name, alignment.Alignment.name, edm::PathSummary.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, PixelEndcapLinkMaker::Item.name, perftools::EdmEventSize::BranchRecord.name, DQMGenericClient::EfficOption.name, FWTableViewManager::TableEntry.name, PixelBarrelLinkMaker::Item.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, EcalLogicID.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, validateAlignments.ParallelMergeJob.name, MEtoEDM< T >::MEtoEDMObject.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, ExpressionHisto< T >.name, XMLProcessor::_loaderBaseConfig.name, cond::persistency::PAYLOAD::HASH.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, TreeCrawler.Package.name, genericValidation.GenericValidation.name, cond::persistency::PAYLOAD::DATA.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, cond::persistency::PAYLOAD::VERSION.name, options.ConnectionHLTMenu.name, MagGeoBuilderFromDDD::volumeHandle.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, DQMGenericClient::ProfileOption.name, dqmoffline::l1t::HistDefinition.name, DQMGenericClient::NormOption.name, emtf::Node.name, h4DSegm.name, PhysicsTools::Calibration::Variable.name, core.TriggerMatchAnalyzer.TriggerMatchAnalyzer.name, FastHFShowerLibrary.name, DQMGenericClient::CDOption.name, CounterChecker.name, cond::TagInfo_t.name, looper.Looper.name, DQMGenericClient::NoFlowOption.name, cond::persistency::IOV::TAG_NAME.name, EDMtoMEConverter.name, cond::persistency::IOV::SINCE.name, TrackerSectorStruct.name, Mapper::definition< ScannerT >.name, cond::persistency::IOV::PAYLOAD_HASH.name, classes.MonitorData.name, cond::persistency::IOV::INSERTION_TIME.name, HistogramManager.name, MuonGeometrySanityCheckPoint.name, classes.OutputData.name, options.HLTProcessOptions.name, h2DSegm.name, core.TriggerBitAnalyzer.TriggerBitAnalyzer.name, nanoaod::FlatTable::Column.name, geometry.Structure.name, config.Analyzer.name, core.autovars.NTupleSubObject.name, DQMNet::WaitObject.name, AlpgenParameterName.name, SiStripMonitorDigi.name, core.autovars.NTupleObject.name, config.Service.name, cond::persistency::TAG_LOG::TAG_NAME.name, cond::persistency::TAG_LOG::EVENT_TIME.name, cond::persistency::TAG_LOG::USER_NAME.name, cond::persistency::TAG_LOG::HOST_NAME.name, cond::persistency::TAG_LOG::COMMAND.name, cond::persistency::TAG_LOG::ACTION.name, cond::persistency::TAG_LOG::USER_TEXT.name, core.autovars.NTupleCollection.name, BPHRecoBuilder::BPHRecoSource.name, BPHRecoBuilder::BPHCompSource.name, personalPlayback.FrameworkJob.name, plotscripts.SawTeethFunction.name, crabFunctions.CrabTask.name, hTMaxCell.name, cscdqm::ParHistoDef.name, BeautifulSoup.Tag.name, SummaryOutputProducer::GenericSummary.name, BeautifulSoup.SoupStrainer.name, and dataset.BaseDataset.run_range.

    def getPrimaryDatasetEntries(self):
        runmin = -1
        runmax = -1
        if self.run_range is not None:
            runmin = self.run_range[0]
            runmax = self.run_range[1]
        return self.findPrimaryDatasetEntries(self.name, runmin, runmax)

Member Data Documentation

dataset.CMSDataset.files

Definition at line 171 of file dataset.py.