dataset.PrivateDataset Class Reference

Inheritance diagram for dataset.PrivateDataset:

Public Member Functions
def	__init__ (self, name, dbsInstance=None)
def	buildListOfFiles (self, pattern='.*root')
def	buildListOfFilesDBS (self, name, dbsInstance)
def	getPrimaryDatasetEntries (self)
Public Member Functions inherited from dataset.BaseDataset
def	__init__ (self, name, user, pattern='.*root', run_range=None, dbsInstance=None)
	def init(self, name, user, pattern='. More...
def	buildListOfBadFiles (self)
def	buildListOfFiles (self, pattern)
def	extractFileSizes (self)
def	getPrimaryDatasetEntries (self)
def	listOfFiles (self)
def	listOfGoodFiles (self)
def	listOfGoodFilesWithPrescale (self, prescale)
def	printFiles (self, abspath=True, info=True)
def	printInfo (self)

Static Public Member Functions
def	findPrimaryDatasetEntries (dataset, dbsInstance, runmin, runmax)
def	findPrimaryDatasetNumFiles (dataset, dbsInstance, runmin, runmax)

Public Attributes
	files
Public Attributes inherited from dataset.BaseDataset
	bad_files
	dbsInstance
	MM. More...
	files
	filesAndSizes
	good_files
	name
	pattern
	primaryDatasetEntries
	MM. More...
	report
	run_range
	user

Detailed Description

Definition at line 336 of file dataset.py.

Constructor & Destructor Documentation

__init__()

def dataset.PrivateDataset.__init__	(		self,
			name,
			dbsInstance = None
	)

Definition at line 338 of file dataset.py.

    def __init__(self, name, dbsInstance=None):
        super(PrivateDataset, self).__init__(name, 'PRIVATE', dbsInstance=dbsInstance)

Member Function Documentation

buildListOfFiles()

def dataset.PrivateDataset.buildListOfFiles	(		self,
			pattern = '.*root'
	)

Definition at line 355 of file dataset.py.

    def buildListOfFiles(self, pattern='.*root'):
        self.files = self.buildListOfFilesDBS(self.name, self.dbsInstance)

buildListOfFilesDBS()

def dataset.PrivateDataset.buildListOfFilesDBS	(		self,
			name,
			dbsInstance
	)

Definition at line 341 of file dataset.py.

References dataset.CMSDataset.findPrimaryDatasetNumFiles(), and dataset.PrivateDataset.findPrimaryDatasetNumFiles().

    def buildListOfFilesDBS(self, name, dbsInstance):
        entries = self.findPrimaryDatasetNumFiles(name, dbsInstance, -1, -1)
        files = []
        dbs = 'das_client.py --query="file dataset=%s instance=prod/%s" --limit=%s' % (name, dbsInstance, entries)
        dbsOut = os.popen(dbs)
        for line in dbsOut:
            if line.find('/store')==-1:
                continue
            line = line.rstrip()
            # print 'line',line
            files.append(line)
        #return ['root://eoscms//eos/cms%s' % f for f in files]
        return files
    

findPrimaryDatasetEntries()

def dataset.PrivateDataset.findPrimaryDatasetEntries (   dataset,   dbsInstance,   runmin,   runmax  )

static

Definition at line 360 of file dataset.py.

References dataset.int, and print().

Referenced by dataset.PrivateDataset.getPrimaryDatasetEntries().

    def findPrimaryDatasetEntries(dataset, dbsInstance, runmin, runmax):

        query, qwhat = dataset, "dataset"
        if "#" in dataset: qwhat = "block"
        if runmin >0 or runmax > 0:
            if runmin == runmax:
                query = "%s run=%d" % (query,runmin)
            else:
                print("WARNING: queries with run ranges are slow in DAS")
                query = "%s run between [%d, %d]" % (query,runmin if runmin > 0 else 1, runmax if runmax > 0 else 999999)
        dbs='das_client.py --query="summary %s=%s instance=prod/%s"'%(qwhat, query, dbsInstance)
        dbsOut = os.popen(dbs).readlines()

        entries = []
        for line in dbsOut:
            line = line.replace('\n','')
            if "nevents" in line:
                entries.append(int(line.split(":")[1]))
        if entries:
            return sum(entries)
        return -1
        

findPrimaryDatasetNumFiles()

def dataset.PrivateDataset.findPrimaryDatasetNumFiles (   dataset,   dbsInstance,   runmin,   runmax  )

static

Definition at line 384 of file dataset.py.

References dataset.int, and print().

Referenced by dataset.PrivateDataset.buildListOfFilesDBS().

    def findPrimaryDatasetNumFiles(dataset, dbsInstance, runmin, runmax):

        query, qwhat = dataset, "dataset"
        if "#" in dataset: qwhat = "block"
        if runmin >0 or runmax > 0:
            if runmin == runmax:
                query = "%s run=%d" % (query,runmin)
            else:
                print("WARNING: queries with run ranges are slow in DAS")
                query = "%s run between [%d, %d]" % (query,runmin if runmin > 0 else 1, runmax if runmax > 0 else 999999)
        dbs='das_client.py --query="summary %s=%s instance=prod/%s"'%(qwhat, query, dbsInstance)
        dbsOut = os.popen(dbs).readlines()
        
        entries = []
        for line in dbsOut:
            line = line.replace('\n','')
            if "nfiles" in line:
                entries.append(int(line.split(":")[1]))
        if entries:
            return sum(entries)
        return -1

getPrimaryDatasetEntries()

def dataset.PrivateDataset.getPrimaryDatasetEntries

(

self

)

Definition at line 406 of file dataset.py.

References dataset.BaseDataset.dbsInstance, dataset.CMSDataset.findPrimaryDatasetEntries(), dataset.PrivateDataset.findPrimaryDatasetEntries(), ElectronMVAID.ElectronMVAID.name, HcalOfflineHarvesting.name, HcalOnlineHarvesting.name, HFRaddamTask.name, LaserTask.name, NoCQTask.name, PedestalTask.name, QIE10Task.name, QIE11Task.name, RecHitTask.name, UMNioTask.name, ZDCTask.name, AlignableObjectId::entry.name, RawTask.name, counter.Counter.name, average.Average.name, TPTask.name, DigiTask.name, histograms.Histograms.name, LEDTask.name, cond::persistency::TAG::NAME.name, cond::persistency::RUN_INFO::RUN_NUMBER.name, TmModule.name, cond::persistency::GTEditorData.name, cond::persistency::GLOBAL_TAG::NAME.name, cond::persistency::RUN_INFO::START_TIME.name, cond::persistency::TAG::TIME_TYPE.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::RUN_INFO::END_TIME.name, cond::persistency::TAG::OBJECT_TYPE.name, core.autovars.NTupleVariable.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, DQMRivetClient::NormOption.name, cond::persistency::GLOBAL_TAG::RELEASE.name, cond::persistency::TAG::SYNCHRONIZATION.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, cond::persistency::TAG::END_OF_VALIDITY.name, MEPSet.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::O2O_RUN::JOB_NAME.name, cms::dd::NameValuePair< T >.name, cond::persistency::TAG::DESCRIPTION.name, cond::persistency::O2O_RUN::START_TIME.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, FWTGeoRecoGeometry::Info.name, cond::persistency::O2O_RUN::END_TIME.name, cond::persistency::TAG::INSERTION_TIME.name, cond::persistency::O2O_RUN::STATUS_CODE.name, cond::persistency::TAG::MODIFICATION_TIME.name, cond::persistency::O2O_RUN::LOG.name, ParameterSet.name, nanoaod::MergeableCounterTable::SingleColumn< T >.name, cond::persistency::TAG::PROTECTION_CODE.name, preexistingValidation.PreexistingValidation.name, OutputMEPSet.name, MEPSetData.name, AlignmentConstraint.name, PixelDCSObject< T >::Item.name, dataset.BaseDataset.name, cms::dd::ValuePair< T, U >.name, personalPlayback.Applet.name, Types._Untracked.name, MagCylinder.name, analyzer.Analyzer.name, DQMRivetClient::LumiOption.name, heppy::ParSet.name, cond::persistency::GTProxyData.name, SingleObjectCondition.name, edm::PathTimingSummary.name, DQMRivetClient::ScaleFactorOption.name, cms::DDAlgoArguments.name, EgHLTOfflineSummaryClient::SumHistBinData.name, Barrel.name, cond::TimeTypeSpecs.name, perftools::EdmEventSize::BranchRecord.name, core.autovars.NTupleObjectType.name, edm::PathSummary.name, EcalLogicID.name, alignment.Alignment.name, lumi::TriggerInfo.name, PixelEndcapLinkMaker::Item.name, XMLProcessor::_loaderBaseConfig.name, FWTableViewManager::TableEntry.name, MEtoEDM< T >::MEtoEDMObject.name, PixelBarrelLinkMaker::Item.name, ExpressionHisto< T >.name, DQMGenericClient::EfficOption.name, TreeCrawler.Package.name, Supermodule.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, genericValidation.GenericValidation.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, options.ConnectionHLTMenu.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, cms::DDParsingContext::CompositeMaterial.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, cond::Tag_t.name, dqmoffline::l1t::HistDefinition.name, DQMGenericClient::ProfileOption.name, FastHFShowerLibrary.name, magneticfield::BaseVolumeHandle.name, nanoaod::MergeableCounterTable::VectorColumn< T >.name, emtf::Node.name, h4DSegm.name, DQMGenericClient::NormOption.name, core.TriggerMatchAnalyzer.TriggerMatchAnalyzer.name, CounterChecker.name, DQMGenericClient::CDOption.name, PhysicsTools::Calibration::Variable.name, cond::TagInfo_t.name, TrackerSectorStruct.name, MuonGeometrySanityCheckPoint.name, DQMGenericClient::NoFlowOption.name, FCDTask.name, cond::persistency::PAYLOAD::HASH.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, looper.Looper.name, Mapper::definition< ScannerT >.name, cond::persistency::PAYLOAD::DATA.name, EDMtoMEConverter.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, cond::persistency::PAYLOAD::VERSION.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, classes.MonitorData.name, HistogramManager.name, classes.OutputData.name, BPHDecayToResResBuilderBase::DZSelect.name, Crystal.name, h2DSegm.name, options.HLTProcessOptions.name, cond::persistency::IOV::TAG_NAME.name, cond::persistency::IOV::SINCE.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, core.TriggerBitAnalyzer.TriggerBitAnalyzer.name, DQMNet::WaitObject.name, AlpgenParameterName.name, config.Analyzer.name, geometry.Structure.name, core.autovars.NTupleSubObject.name, Capsule.name, core.autovars.NTupleObject.name, Ceramic.name, SiStripMonitorDigi.name, config.Service.name, BulkSilicon.name, APD.name, core.autovars.NTupleCollection.name, nanoaod::FlatTable::Column.name, BPHRecoBuilder::BPHRecoSource.name, BPHRecoBuilder::BPHCompSource.name, StraightTrackAlignment::RPSetPlots.name, cond::persistency::TAG_AUTHORIZATION::TAG_NAME.name, cond::persistency::TAG_AUTHORIZATION::ACCESS_TYPE.name, cond::persistency::TAG_AUTHORIZATION::CREDENTIAL.name, cond::persistency::TAG_AUTHORIZATION::CREDENTIAL_TYPE.name, InnerLayerVolume.name, cond::payloadInspector::TagReference.name, cond::persistency::TAG_LOG::TAG_NAME.name, cond::persistency::TAG_LOG::EVENT_TIME.name, cond::persistency::TAG_LOG::USER_NAME.name, cond::persistency::TAG_LOG::HOST_NAME.name, cond::persistency::TAG_LOG::COMMAND.name, cond::persistency::TAG_LOG::ACTION.name, cond::persistency::TAG_LOG::USER_TEXT.name, personalPlayback.FrameworkJob.name, Grid.name, trklet::TrackletConfigBuilder::DTCinfo.name, Grille.name, BackPipe.name, plotscripts.SawTeethFunction.name, PatchPanel.name, BackCoolTank.name, DryAirTube.name, crabFunctions.CrabTask.name, MBCoolTube.name, MBManif.name, cscdqm::ParHistoDef.name, hTMaxCell.name, BeautifulSoup.Tag.name, SummaryOutputProducer::GenericSummary.name, BeautifulSoup.SoupStrainer.name, and dataset.BaseDataset.run_range.

    def getPrimaryDatasetEntries(self):
        runmin = -1
        runmax = -1
        if self.run_range is not None:
            runmin = self.run_range[0]
            runmax = self.run_range[1]
        return self.findPrimaryDatasetEntries(self.name, self.dbsInstance, runmin, runmax)

Member Data Documentation

files

dataset.PrivateDataset.files

Definition at line 356 of file dataset.py.