dataset.Dataset Class Reference

Inheritance diagram for dataset.Dataset:

Public Member Functions
def	__init__
def	__init__
def	buildListOfBadFiles
def	buildListOfFiles
def	convertTimeToRun
def	createdatasetfile_hippy
def	datasetSnippet
def	dataType
def	dump_cff
def	extractFileSizes
def	fileInfoList
def	fileList
def	forcerunrange
def	getForceRunRangeFunction
def	getPrimaryDatasetEntries
def	magneticField
def	magneticFieldForRun
def	name
def	parentDataset
def	predefined
def	printInfo
def	runList
Public Member Functions inherited from dataset.BaseDataset
def	__init__
	def init(self, name, user, pattern='. More...
def	buildListOfBadFiles
def	buildListOfFiles
def	extractFileSizes
def	getPrimaryDatasetEntries
def	listOfFiles
def	listOfGoodFiles
def	listOfGoodFilesWithPrescale
def	printFiles
def	printInfo

Static Public Member Functions
def	getrunnumberfromfilename

Public Attributes
	bad_files
	castorDir
	files
	filesAndSizes
	good_files
	lfnDir
	maskExists
	report
Public Attributes inherited from dataset.BaseDataset
	bad_files
	dbsInstance
	MM. More...
	files
	filesAndSizes
	good_files
	name
	pattern
	primaryDatasetEntries
	MM. More...
	report
	run_range
	user

Static Public Attributes
tuple	dasData = das_client.get_data(dasQuery, dasLimit)
tuple	error = self.__findInJson(jsondict,["data","error"])
	error = None
int	i = 0
tuple	jsondict = json.loads( dasData )
	jsondict = dasData
string	jsonfile = "das_query_output_%i.txt"
	jsonfile = jsonfile%i
tuple	jsonstr = self.__findInJson(jsondict,"reason")
string	msg = "The DAS query returned an error. The output is very long, and has been stored in:\n"
tuple	theFile = open( jsonfile, "w" )

Private Member Functions
def	__chunks
def	__createSnippet
def	__dateString
def	__datetime
def	__fileListSnippet
def	__find_ge
def	__find_lt
def	__findInJson
def	__getData
def	__getDataType
def	__getFileInfoList
def	__getMagneticField
def	__getMagneticFieldForRun
def	__getParentDataset
def	__getRunList
def	__lumiSelectionSnippet

Private Attributes
	__alreadyStored
	__cmssw
	__cmsswrelease
	__dasLimit
	__dataType
	__fileInfoList
	__fileList
	__filename
	__firstusedrun
	__lastusedrun
	__magneticField
	__name
	__official
	__origName
	__parentDataset
	__parentFileInfoList
	__parentFileList
	__predefined
	__runList

Static Private Attributes
tuple	__dummy_source_template

Detailed Description

Definition at line 14 of file dataset.py.

Constructor & Destructor Documentation

def dataset.Dataset.__init__	(		self,
			datasetName,
			dasLimit = 0,
			tryPredefinedFirst = True,
			cmssw = os.environ["CMSSW_BASE"],
			cmsswrelease = os.environ["CMSSW_RELEASE_BASE"]
	)

Definition at line 16 of file dataset.py.

def dataset.Dataset.__init__	(		self,
			name,
			user,
			pattern = '.*root'
	)

Definition at line 264 of file dataset.py.

Member Function Documentation

def dataset.Dataset.__chunks (   self,   theList,   n  )

private

Yield successive n-sized chunks from theList.

Definition at line 79 of file dataset.py.

def dataset.Dataset.__createSnippet (   self,   jsonPath = None,   begin = None,   end = None,   firstRun = None,   lastRun = None,   repMap = None,   crab = False,   parent = False  )

private

Definition at line 230 of file dataset.py.

def dataset.Dataset.__dateString (   self,   date  )

private

Definition at line 621 of file dataset.py.

def dataset.Dataset.__datetime (   self,   stringForDas  )

private

Definition at line 612 of file dataset.py.

def dataset.Dataset.__fileListSnippet (   self,   crab = False,   parent = False,   firstRun = None,   lastRun = None,   forcerunselection = False  )

private

Definition at line 208 of file dataset.py.

def dataset.Dataset.__find_ge (   self,   a,   x  )

private

Definition at line 282 of file dataset.py.

def dataset.Dataset.__find_lt (   self,   a,   x  )

private

Definition at line 275 of file dataset.py.

def dataset.Dataset.__findInJson (   self,   jsondict,   strings  )

private

Definition at line 289 of file dataset.py.

def dataset.Dataset.__getData (   self,   dasQuery,   dasLimit = 0  )

private

Definition at line 341 of file dataset.py.

def dataset.Dataset.__getDataType (   self )

private

Definition at line 373 of file dataset.py.

def dataset.Dataset.__getFileInfoList (   self,   dasLimit,   parent = False  )

private

Definition at line 533 of file dataset.py.

def dataset.Dataset.__getMagneticField (   self )

private

Definition at line 408 of file dataset.py.

def dataset.Dataset.__getMagneticFieldForRun (   self,   run = -1,   tolerance = 0.5  )

private

For MC, this returns the same as the previous function.
   For data, it gets the magnetic field from the runs.  This is important for
   deciding which template to use for offlinevalidation

Definition at line 479 of file dataset.py.

def dataset.Dataset.__getParentDataset (   self )

private

Definition at line 398 of file dataset.py.

def dataset.Dataset.__getRunList (   self )

private

Definition at line 599 of file dataset.py.

def dataset.Dataset.__lumiSelectionSnippet (   self,   jsonPath = None,   firstRun = None,   lastRun = None  )

private

Definition at line 115 of file dataset.py.

def dataset.Dataset.buildListOfBadFiles

(

self

)

fills the list of bad files from the IntegrityCheck log.

When the integrity check file is not available,
files are considered as good.

Definition at line 275 of file dataset.py.

def dataset.Dataset.buildListOfFiles	(		self,
			pattern = '.*root'
	)

fills list of files, taking all root files matching the pattern in the castor dir

Definition at line 271 of file dataset.py.

def dataset.Dataset.convertTimeToRun	(		self,
			begin = None,
			end = None,
			firstRun = None,
			lastRun = None,
			shortTuple = True
	)

Definition at line 626 of file dataset.py.

def dataset.Dataset.createdatasetfile_hippy	(		self,
			filename,
			filesperjob,
			firstrun,
			lastrun
	)

Definition at line 831 of file dataset.py.

def dataset.Dataset.datasetSnippet	(		self,
			jsonPath = None,
			begin = None,
			end = None,
			firstRun = None,
			lastRun = None,
			crab = False,
			parent = False
	)

Definition at line 710 of file dataset.py.

def dataset.Dataset.dataType

(

self

)

Definition at line 691 of file dataset.py.

def dataset.Dataset.dump_cff	(		self,
			outName = None,
			jsonPath = None,
			begin = None,
			end = None,
			firstRun = None,
			lastRun = None,
			parent = False
	)

Definition at line 767 of file dataset.py.

def dataset.Dataset.extractFileSizes

(

self

)

Get the file size for each file, from the eos ls -l command.

Definition at line 306 of file dataset.py.

def dataset.Dataset.fileInfoList	(		self,
			parent = False
	)

Definition at line 901 of file dataset.py.

def dataset.Dataset.fileList	(		self,
			parent = False,
			firstRun = None,
			lastRun = None,
			forcerunselection = False
	)

Definition at line 863 of file dataset.py.

def dataset.Dataset.forcerunrange	(		self,
			firstRun,
			lastRun,
			s
	)

s must be in the format run1:lum1-run2:lum2

Definition at line 311 of file dataset.py.

def dataset.Dataset.getForceRunRangeFunction	(		self,
			firstRun,
			lastRun
	)

Definition at line 336 of file dataset.py.

def dataset.Dataset.getPrimaryDatasetEntries

(

self

)

Definition at line 326 of file dataset.py.

def dataset.Dataset.getrunnumberfromfilename (   filename )

static

Definition at line 837 of file dataset.py.

def dataset.Dataset.magneticField

(

self

)

Definition at line 696 of file dataset.py.

def dataset.Dataset.magneticFieldForRun	(		self,
			run = -1
	)

Definition at line 701 of file dataset.py.

def dataset.Dataset.name

(

self

)

Definition at line 904 of file dataset.py.

Referenced by VIDSelectorBase.VIDSelectorBase.initialize().

def dataset.Dataset.parentDataset

(

self

)

Definition at line 704 of file dataset.py.

def dataset.Dataset.predefined

(

self

)

Definition at line 907 of file dataset.py.

def dataset.Dataset.printInfo

(

self

)

Definition at line 321 of file dataset.py.

def dataset.Dataset.runList

(

self

)

Definition at line 910 of file dataset.py.

Member Data Documentation

dataset.Dataset.__alreadyStored

private

Definition at line 23 of file dataset.py.

dataset.Dataset.__cmssw

private

Definition at line 24 of file dataset.py.

dataset.Dataset.__cmsswrelease

private

Definition at line 25 of file dataset.py.

dataset.Dataset.__dasLimit

private

Definition at line 19 of file dataset.py.

dataset.Dataset.__dataType

private

Definition at line 76 of file dataset.py.

tuple dataset.Dataset.__dummy_source_template

staticprivate

Initial value:

= ("readFiles = cms.untracked.vstring()\n"
                               "secFiles = cms.untracked.vstring()\n"
                               "%(process)ssource = cms.Source(\"PoolSource\",\n"
                               "%(tab)s                    secondaryFileNames ="
                               "secFiles,\n"
                               "%(tab)s                    fileNames = readFiles\n"
                               ")\n"
                               "readFiles.extend(['dummy_File.root'])\n"
                               "%(process)smaxEvents = cms.untracked.PSet( "
                               "input = cms.untracked.int32(%(nEvents)s) )\n"
                               "%(skipEventsString)s\n")

Definition at line 103 of file dataset.py.

dataset.Dataset.__fileInfoList

private

Definition at line 21 of file dataset.py.

dataset.Dataset.__fileList

private

Definition at line 20 of file dataset.py.

dataset.Dataset.__filename

private

Definition at line 53 of file dataset.py.

Referenced by csvReporter.csvReporter.writeRow(), and csvReporter.csvReporter.writeRows().

dataset.Dataset.__firstusedrun

private

Definition at line 26 of file dataset.py.

dataset.Dataset.__lastusedrun

private

Definition at line 27 of file dataset.py.

dataset.Dataset.__magneticField

private

Definition at line 77 of file dataset.py.

dataset.Dataset.__name

private

Definition at line 17 of file dataset.py.

dataset.Dataset.__official

private

Definition at line 34 of file dataset.py.

dataset.Dataset.__origName

private

Definition at line 18 of file dataset.py.

dataset.Dataset.__parentDataset

private

Definition at line 28 of file dataset.py.

dataset.Dataset.__parentFileInfoList

private

Definition at line 30 of file dataset.py.

dataset.Dataset.__parentFileList

private

Definition at line 29 of file dataset.py.

dataset.Dataset.__predefined

private

Definition at line 50 of file dataset.py.

dataset.Dataset.__runList

private

Definition at line 22 of file dataset.py.

dataset.Dataset.bad_files

Definition at line 282 of file dataset.py.

dataset.Dataset.castorDir

Definition at line 266 of file dataset.py.

tuple dataset.Dataset.dasData = das_client.get_data(dasQuery, dasLimit)

static

Definition at line 342 of file dataset.py.

tuple dataset.Dataset.error = self.__findInJson(jsondict,["data","error"])

static

Definition at line 349 of file dataset.py.

dataset.Dataset.error = None

static

Definition at line 351 of file dataset.py.

dataset.Dataset.files

Definition at line 273 of file dataset.py.

dataset.Dataset.filesAndSizes

Definition at line 311 of file dataset.py.

dataset.Dataset.good_files

Definition at line 283 of file dataset.py.

int dataset.Dataset.i = 0

static

Definition at line 359 of file dataset.py.

tuple dataset.Dataset.jsondict = json.loads( dasData )

static

Definition at line 344 of file dataset.py.

dataset.Dataset.jsondict = dasData

static

Definition at line 346 of file dataset.py.

string dataset.Dataset.jsonfile = "das_query_output_%i.txt"

static

Definition at line 358 of file dataset.py.

dataset.Dataset.jsonfile = jsonfile%i

static

Definition at line 362 of file dataset.py.

tuple dataset.Dataset.jsonstr = self.__findInJson(jsondict,"reason")

static

Definition at line 354 of file dataset.py.

dataset.Dataset.lfnDir

Definition at line 265 of file dataset.py.

dataset.Dataset.maskExists

Definition at line 267 of file dataset.py.

string dataset.Dataset.msg = "The DAS query returned an error. The output is very long, and has been stored in:\n"

static

Definition at line 366 of file dataset.py.

dataset.Dataset.report

Definition at line 268 of file dataset.py.

tuple dataset.Dataset.theFile = open( jsonfile, "w" )

static

Definition at line 363 of file dataset.py.