dataset.Dataset Class Reference

Inheritance diagram for dataset.Dataset:

Public Member Functions
def	__init__ (self, datasetname, dasinstance=defaultdasinstance)
def	__init__ (self, name, user, pattern='.*root')
def	buildListOfBadFiles (self)
def	buildListOfFiles (self, pattern='.*root')
def	extractFileSizes (self)
def	getfiles (self, usecache)
def	getPrimaryDatasetEntries (self)
def	headercomment (self)
def	printInfo (self)
Public Member Functions inherited from dataset.BaseDataset
def	__init__ (self, name, user, pattern='.*root', run_range=None, dbsInstance=None)
	def init(self, name, user, pattern='. More...
def	buildListOfBadFiles (self)
def	buildListOfFiles (self, pattern)
def	extractFileSizes (self)
def	getPrimaryDatasetEntries (self)
def	listOfFiles (self)
def	listOfGoodFiles (self)
def	listOfGoodFilesWithPrescale (self, prescale)
def	printFiles (self, abspath=True, info=True)
def	printInfo (self)
Public Member Functions inherited from dataset.DatasetBase
def	getfiles (self, usecache)
def	headercomment (self)
def	writefilelist_hippy (self, firstrun, lastrun, runs, eventsperjob, maxevents, outputfile, usecache=True)
def	writefilelist_validation (self, firstrun, lastrun, runs, maxevents, outputfile=None, usecache=True)

Public Attributes
	bad_files
	castorDir
	dasinstance
	datasetname
	filenamebase
	files
	filesAndSizes
	good_files
	lfnDir
	maskExists
	official
	report
Public Attributes inherited from dataset.BaseDataset
	bad_files
	dbsInstance
	MM. More...
	files
	filesAndSizes
	good_files
	name
	pattern
	primaryDatasetEntries
	MM. More...
	report
	run_range
	user

Detailed Description

Definition at line 198 of file dataset.py.

Constructor & Destructor Documentation

__init__() [1/2]

def dataset.Dataset.__init__	(		self,
			datasetname,
			dasinstance = defaultdasinstance
	)

Definition at line 199 of file dataset.py.

Referenced by dataset.Dataset.__init__().

  def __init__(self, datasetname, dasinstance=defaultdasinstance):
    self.datasetname = datasetname
    if re.match(r'/.+/.+/.+', datasetname):
      self.official = True
      self.filenamebase = "Dataset" + self.datasetname.replace("/","_")
    else:
      self.official = False
      self.filenamebase = datasetname

    self.dasinstance = dasinstance

__init__() [2/2]

def dataset.Dataset.__init__	(		self,
			name,
			user,
			pattern = '.*root'
	)

Definition at line 267 of file dataset.py.

References dataset.Dataset.__init__().

    def __init__(self, name, user, pattern='.*root'):
        self.lfnDir = castorBaseDir(user) + name
        self.castorDir = castortools.lfnToCastor( self.lfnDir )
        self.maskExists = False
        self.report = None
        super(Dataset, self).__init__(name, user, pattern)
        

Member Function Documentation

buildListOfBadFiles()

def dataset.Dataset.buildListOfBadFiles

(

self

)

fills the list of bad files from the IntegrityCheck log.

When the integrity check file is not available,
files are considered as good.

Definition at line 278 of file dataset.py.

    def buildListOfBadFiles(self):
        '''fills the list of bad files from the IntegrityCheck log.

        When the integrity check file is not available,
        files are considered as good.'''
        mask = "IntegrityCheck"
           
        self.bad_files = {}
        self.good_files = []

        file_mask = castortools.matchingFiles(self.castorDir, '^%s_.*\.txt$' % mask)
        if file_mask:
            # here to avoid circular dependency
            from .edmIntegrityCheck import PublishToFileSystem
            p = PublishToFileSystem(mask)
            report = p.get(self.castorDir)
            if report is not None and report:
                self.maskExists = True
                self.report = report
                dup = report.get('ValidDuplicates',{})
                for name, status in report['Files'].items():
                    # print name, status
                    if not status[0]:
                        self.bad_files[name] = 'MarkedBad'
                    elif name in dup:
                        self.bad_files[name] = 'ValidDup'
                    else:
                        self.good_files.append( name )
        else:
            raise IntegrityCheckError( "ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found" )

buildListOfFiles()

def dataset.Dataset.buildListOfFiles	(		self,
			pattern = '.*root'
	)

fills list of files, taking all root files matching the pattern in the castor dir

Definition at line 274 of file dataset.py.

    def buildListOfFiles(self, pattern='.*root'):
        '''fills list of files, taking all root files matching the pattern in the castor dir'''
        self.files = castortools.matchingFiles( self.castorDir, pattern )
                             

extractFileSizes()

def dataset.Dataset.extractFileSizes

(

self

)

Get the file size for each file, from the eos ls -l command.

Definition at line 309 of file dataset.py.

References dataset.EOSDataset.castorDir, and dataset.Dataset.castorDir.

    def extractFileSizes(self):
        '''Get the file size for each file, from the eos ls -l command.'''
        # EOS command does not work in tier3
        lsout = castortools.runXRDCommand(self.castorDir,'dirlist')[0]
        lsout = lsout.split('\n')
        self.filesAndSizes = {}
        for entry in lsout:
            values = entry.split()
            if( len(values) != 5):
                continue
            # using full abs path as a key.
            file = '/'.join([self.lfnDir, values[4].split("/")[-1]])
            size = values[1]
            self.filesAndSizes[file] = size 
         

getfiles()

def dataset.Dataset.getfiles	(		self,
			usecache
	)

Definition at line 211 of file dataset.py.

References dataset.Dataset.dasinstance, dataset.dasquery(), dataset.Dataset.datasetname, dataset.Dataset.filenamebase, dataset.findinjson(), dataset.int, and print().

  def getfiles(self, usecache):
    filename = os.path.join(os.environ["CMSSW_BASE"], "src", "Alignment", "CommonAlignment", "data", self.filenamebase+".csv")
    if not usecache:
      try:
        os.remove(filename)
      except OSError as e:
        if os.path.exists(filename):
          raise

    result = []
    try:
      with open(filename) as f:
        for row in csv.DictReader(f):
          result.append(DataFile(**row))
        return result
    except IOError:
      pass

    query = "file dataset={} instance={} detail=true | grep file.name, file.nevents".format(self.datasetname, self.dasinstance)
    dasoutput = dasquery(query)
    if not dasoutput:
      raise DatasetError("No files are available for the dataset '{}'. This can be "
                         "due to a typo or due to a DAS problem. Please check the "
                         "spelling of the dataset and/or try again.".format(datasetname))
    result = [DataFile(findinjson(_, "file", "name"), findinjson(_, "file", "nevents")) for _ in dasoutput if int(findinjson(_, "file", "nevents"))]
    try:
      with open(filename, "w") as f:
        writer = csv.DictWriter(f, ("filename", "nevents", "runs"))
        writer.writeheader()
        for datafile in result:
          writer.writerow(datafile.getdict())
    except Exception as e:
      print("Couldn't write the dataset csv file:\n\n{}".format(e))
    return result

getPrimaryDatasetEntries()

def dataset.Dataset.getPrimaryDatasetEntries

(

self

)

Definition at line 329 of file dataset.py.

References dataset.int, runall.testit.report, WorkFlowRunner.WorkFlowRunner.report, ALIUtils.report, and dataset.BaseDataset.report.

    def getPrimaryDatasetEntries(self):
        if self.report is not None and self.report:
            return int(self.report.get('PrimaryDatasetEntries',-1))
        return -1

headercomment()

def dataset.Dataset.headercomment

(

self

)

Definition at line 247 of file dataset.py.

References dataset.Dataset.datasetname.

  def headercomment(self):
    return self.datasetname

printInfo()

def dataset.Dataset.printInfo

(

self

)

Definition at line 324 of file dataset.py.

References dataset.EOSDataset.castorDir, dataset.Dataset.castorDir, dataset.Dataset.lfnDir, ElectronMVAID.ElectronMVAID.name, AlignableObjectId::entry.name, average.Average.name, counter.Counter.name, histograms.Histograms.name, TmModule.name, cond::persistency::RUN_INFO::RUN_NUMBER.name, cond::persistency::TAG::NAME.name, cond::persistency::GTEditorData.name, cond::persistency::GLOBAL_TAG::NAME.name, cond::persistency::TAG::TIME_TYPE.name, cond::persistency::RUN_INFO::START_TIME.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, cond::persistency::RUN_INFO::END_TIME.name, core.autovars.NTupleVariable.name, cond::persistency::TAG::OBJECT_TYPE.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, cond::persistency::TAG::SYNCHRONIZATION.name, cond::persistency::GLOBAL_TAG::RELEASE.name, DQMRivetClient::NormOption.name, cond::persistency::TAG::END_OF_VALIDITY.name, MEPSet.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, cond::persistency::O2O_RUN::JOB_NAME.name, cond::persistency::TAG::DESCRIPTION.name, cms::dd::NameValuePair< T >.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::O2O_RUN::START_TIME.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, cond::persistency::O2O_RUN::END_TIME.name, cond::persistency::TAG::INSERTION_TIME.name, FWTGeoRecoGeometry::Info.name, cond::persistency::O2O_RUN::STATUS_CODE.name, cond::persistency::TAG::MODIFICATION_TIME.name, cond::persistency::O2O_RUN::LOG.name, ParameterSet.name, nanoaod::MergeableCounterTable::SingleColumn< T >.name, cond::persistency::TAG::PROTECTION_CODE.name, OutputMEPSet.name, PixelDCSObject< T >::Item.name, dataset.BaseDataset.name, AlignmentConstraint.name, cms::dd::ValuePair< T, U >.name, personalPlayback.Applet.name, Types._Untracked.name, MagCylinder.name, analyzer.Analyzer.name, heppy::ParSet.name, DQMRivetClient::LumiOption.name, cond::persistency::GTProxyData.name, SingleObjectCondition.name, edm::PathTimingSummary.name, DQMRivetClient::ScaleFactorOption.name, cms::DDAlgoArguments.name, EgHLTOfflineSummaryClient::SumHistBinData.name, Barrel.name, perftools::EdmEventSize::BranchRecord.name, cond::TimeTypeSpecs.name, core.autovars.NTupleObjectType.name, EcalLogicID.name, edm::PathSummary.name, lumi::TriggerInfo.name, XMLProcessor::_loaderBaseConfig.name, PixelEndcapLinkMaker::Item.name, MEtoEDM< T >::MEtoEDMObject.name, FWTableViewManager::TableEntry.name, PixelBarrelLinkMaker::Item.name, DQMGenericClient::EfficOption.name, ExpressionHisto< T >.name, TreeCrawler.Package.name, Supermodule.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, options.ConnectionHLTMenu.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, cms::DDParsingContext::CompositeMaterial.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, cond::Tag_t.name, dqmoffline::l1t::HistDefinition.name, DQMGenericClient::ProfileOption.name, magneticfield::BaseVolumeHandle.name, nanoaod::MergeableCounterTable::VectorColumn< T >.name, FastHFShowerLibrary.name, emtf::Node.name, h4DSegm.name, DQMGenericClient::NormOption.name, core.TriggerMatchAnalyzer.TriggerMatchAnalyzer.name, DQMGenericClient::CDOption.name, CounterChecker.name, cond::TagInfo_t.name, TrackerSectorStruct.name, MuonGeometrySanityCheckPoint.name, PhysicsTools::Calibration::Variable.name, cond::persistency::PAYLOAD::HASH.name, EDMtoMEConverter.name, DQMGenericClient::NoFlowOption.name, looper.Looper.name, Mapper::definition< ScannerT >.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, cond::persistency::PAYLOAD::DATA.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, cond::persistency::PAYLOAD::VERSION.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, classes.MonitorData.name, HistogramManager.name, classes.OutputData.name, BPHDecayToResResBuilderBase::DZSelect.name, Crystal.name, h2DSegm.name, options.HLTProcessOptions.name, cond::persistency::IOV::TAG_NAME.name, cond::persistency::IOV::SINCE.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, DQMNet::WaitObject.name, core.TriggerBitAnalyzer.TriggerBitAnalyzer.name, AlpgenParameterName.name, config.Analyzer.name, geometry.Structure.name, core.autovars.NTupleSubObject.name, Capsule.name, core.autovars.NTupleObject.name, Ceramic.name, SiStripMonitorDigi.name, config.Service.name, BulkSilicon.name, APD.name, core.autovars.NTupleCollection.name, BPHRecoBuilder::BPHRecoSource.name, nanoaod::FlatTable::Column.name, BPHRecoBuilder::BPHCompSource.name, StraightTrackAlignment::RPSetPlots.name, cond::persistency::TAG_AUTHORIZATION::TAG_NAME.name, lutNN::LutNeuronLayerFixedPoint< input_I, input_F, inputSize, lut_I, lut_F, neurons, output_I >.name, cond::persistency::TAG_AUTHORIZATION::ACCESS_TYPE.name, cond::persistency::TAG_AUTHORIZATION::CREDENTIAL.name, cond::persistency::TAG_AUTHORIZATION::CREDENTIAL_TYPE.name, InnerLayerVolume.name, cond::payloadInspector::TagReference.name, cond::persistency::TAG_LOG::TAG_NAME.name, cond::persistency::TAG_LOG::EVENT_TIME.name, cond::persistency::TAG_LOG::USER_NAME.name, cond::persistency::TAG_LOG::HOST_NAME.name, cond::persistency::TAG_LOG::COMMAND.name, cond::persistency::TAG_LOG::ACTION.name, cond::persistency::TAG_LOG::USER_TEXT.name, lutNN::LutNetworkFixedPointRegression2Outputs< input_I, input_F, inputSize, layer1_lut_I, layer1_lut_F, layer1_neurons, layer1_output_I, layer2_input_I, layer2_lut_I, layer2_lut_F, layer2_neurons, layer3_input_I, layer3_0_inputCnt, layer3_0_lut_I, layer3_0_lut_F, output0_I, output0_F, layer3_1_inputCnt, layer3_1_lut_I, layer3_1_lut_F, output1_I, output1_F >.name, personalPlayback.FrameworkJob.name, Grid.name, trklet::TrackletConfigBuilder::DTCinfo.name, Grille.name, BackPipe.name, plotscripts.SawTeethFunction.name, PatchPanel.name, BackCoolTank.name, DryAirTube.name, crabFunctions.CrabTask.name, MBCoolTube.name, MBManif.name, cscdqm::ParHistoDef.name, hTMaxCell.name, SummaryOutputProducer::GenericSummary.name, and print().

    def printInfo(self):
        print('sample      :  ' + self.name)
        print('LFN         :  ' + self.lfnDir)
        print('Castor path :  ' + self.castorDir)

Member Data Documentation

bad_files

dataset.Dataset.bad_files

Definition at line 285 of file dataset.py.

castorDir

dataset.Dataset.castorDir

Definition at line 269 of file dataset.py.

Referenced by dataset.Dataset.extractFileSizes(), and dataset.Dataset.printInfo().

dasinstance

dataset.Dataset.dasinstance

Definition at line 208 of file dataset.py.

Referenced by dataset.Dataset.getfiles().

datasetname

dataset.Dataset.datasetname

Definition at line 200 of file dataset.py.

Referenced by dataset.Dataset.getfiles(), and dataset.Dataset.headercomment().

filenamebase

dataset.Dataset.filenamebase

Definition at line 203 of file dataset.py.

Referenced by dataset.Dataset.getfiles().

files

dataset.Dataset.files

Definition at line 276 of file dataset.py.

filesAndSizes

dataset.Dataset.filesAndSizes

Definition at line 314 of file dataset.py.

good_files

dataset.Dataset.good_files

Definition at line 286 of file dataset.py.

lfnDir

dataset.Dataset.lfnDir

Definition at line 268 of file dataset.py.

Referenced by dataset.Dataset.printInfo().

maskExists

dataset.Dataset.maskExists

Definition at line 270 of file dataset.py.

official

dataset.Dataset.official

Definition at line 202 of file dataset.py.

report

dataset.Dataset.report

Definition at line 271 of file dataset.py.

Referenced by addOnTests.testit.run().