Classes
class	BaseDataset

class	CMSDataset

class	DataFile

class	Dataset

class	DatasetBase

class	DatasetError

class	EOSDataset

class	IntegrityCheckError

class	LocalDataset

class	MultipleDatasets

class	PrivateDataset

class	RunRange

Functions
def	createDataset (user, dataset, pattern, readcache=False, basedir=None, run_range=None)

def	createMyDataset (user, dataset, pattern, dbsInstance, readcache=False)
	if user == 'CMS': data = CMSDataset( dataset ) elif user == 'LOCAL': if basedir is None: basedir = os.environ['CMGLOCALBASEDIR'] data = LocalDataset( dataset, basedir, pattern ) else: data = Dataset( user, dataset, pattern ) More...

def	dasquery (dasQuery, dasLimit=0)

def	findinjson (jsondict, strings)

def	getDatasetFromCache (cachename)

def	getrunnumbersfromfile (filename, trydas=True, allowunknown=False, dasinstance=defaultdasinstance)

def	writeDatasetToCache (cachename, dataset)

Variables
	abspath

	action

	args

	data

	dataset = Dataset( datasetName )

string	datasetName = '/MinimumBias/Run2012D-TkAlMinBias-v1/ALCARECO'

	default

string	defaultdasinstance = "prod/global"

	dest

	end

	firstRun

	help

	info

	int

tuple	jsonFile

	jsonPath

	name

	options

	outName

	parser

	run_range

	type

	usage

	user

string	validationfooter

string	validationheader

Function Documentation

def dataset.createDataset	(	user,
		dataset,
		pattern,
		readcache = `False`,
		basedir = `None`,
		run_range = `None`
	)

Definition at line 430 of file dataset.py.

References getDatasetFromCache(), and writeDatasetToCache().

Referenced by datasetToSource.datasetToSource(), production_tasks.CheckDatasetExists.run(), production_tasks.SourceCFG.run(), and writeDatasetToCache().

                    basedir = None, run_range = None):
     
     
     def cacheFileName(data, user, pattern):
         return '{user}%{name}%{pattern}.pck'.format( user = user, name = data.replace('/','_'), pattern = pattern)
 
     def writeCache(dataset):
         writeDatasetToCache( cacheFileName(dataset.name, dataset.user, dataset.pattern), dataset )
 
     def readCache(data, user, pattern):
         return getDatasetFromCache( cacheFileName(data, user, pattern) )
 
     if readcache:
         try:
             data = readCache(dataset, user, pattern)
         except IOError:
             readcache = False
     if not readcache:
         if user == 'CMS':
             data = CMSDataset( dataset , run_range = run_range)
             info = False
         elif user == 'LOCAL':
             data = LocalDataset( dataset, basedir, pattern)
             info = False
         elif user == 'EOS':
             data = EOSDataset(dataset, basedir, pattern)
             info = False
         else:
             data = Dataset( dataset, user, pattern)
         writeCache(data)

def dataset.createMyDataset	(	user,
		dataset,
		pattern,
		dbsInstance,
		readcache = `False`
	)

if user == 'CMS': data = CMSDataset( dataset ) elif user == 'LOCAL': if basedir is None: basedir = os.environ['CMGLOCALBASEDIR'] data = LocalDataset( dataset, basedir, pattern ) else: data = Dataset( user, dataset, pattern )

MM

Definition at line 471 of file dataset.py.

References join().

Referenced by datasetToSource.myDatasetToSource().

 def createMyDataset( user, dataset, pattern, dbsInstance, readcache=False):
 
     cachedir =  '/'.join( [os.environ['HOME'],'.cmgdataset'])
 
     def cacheFileName(data, user, dbsInstance, pattern):
         cf =  data.replace('/','_')
         name = '{dir}/{user}%{dbsInstance}%{name}%{pattern}.pck'.format(
             dir = cachedir,
             user = user,
             dbsInstance = dbsInstance,
             name = cf,
             pattern = pattern)
         return name
 
     def writeCache(dataset):
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
         cachename = cacheFileName(dataset.name,
                                   dataset.user,
                                   dataset.dbsInstance,
                                   dataset.pattern)
         pckfile = open( cachename, 'w')
         pickle.dump(dataset, pckfile)
 
     def readCache(data, user, dbsInstance, pattern):
         cachename = cacheFileName(data, user, dbsInstance, pattern)
         
         pckfile = open( cachename)
         dataset = pickle.load(pckfile)
         #print 'reading cache'                                                                                                                                                                   
         return dataset
 
     if readcache:
         try:
             data = readCache(dataset, user, dbsInstance, pattern)    
         except IOError:
             readcache = False
     if not readcache:
         if user == 'PRIVATE':
             data = PrivateDataset( dataset, dbsInstance )
             info = False
         writeCache(data)
     return data

def dataset.dasquery	(	dasQuery,
		dasLimit = `0`
	)

Definition at line 27 of file dataset.py.

References findinjson(), and str.

Referenced by dataset.Dataset.getfiles(), and getrunnumbersfromfile().

 def dasquery(dasQuery, dasLimit=0):
   dasData = das_client.get_data(dasQuery, dasLimit)
   if isinstance(dasData, str):
     jsondict = json.loads( dasData )
   else:
     jsondict = dasData
   # Check, if the DAS query fails
   try:
     error = findinjson(jsondict, "data","error")
   except KeyError:
     error = None
   if error or findinjson(jsondict, "status") != 'ok' or "data" not in jsondict:
     try:
       jsonstr = findinjson(jsondict, "reason")
     except KeyError: 
       jsonstr = str(jsondict)
     if len(jsonstr) > 10000:
       jsonfile = "das_query_output_%i.txt"
       i = 0
       while os.path.lexists(jsonfile % i):
         i += 1
       jsonfile = jsonfile % i
       theFile = open( jsonfile, "w" )
       theFile.write( jsonstr )
       theFile.close()
       msg = "The DAS query returned an error.  The output is very long, and has been stored in:\n" + jsonfile
     else:
       msg = "The DAS query returned a error.  Here is the output\n" + jsonstr
     msg += "\nIt's possible that this was a server error.  If so, it may work if you try again later"
     raise DatasetError(msg)
   return findinjson(jsondict, "data")
 

def dataset.findinjson	(	jsondict,
		strings
	)

Definition at line 95 of file dataset.py.

Referenced by dasquery(), dataset.Dataset.getfiles(), and getrunnumbersfromfile().

 def findinjson(jsondict, *strings):
   if len(strings) == 0:
     return jsondict
   if isinstance(jsondict,dict):
     if strings[0] in jsondict:
       try:
         return findinjson(jsondict[strings[0]], *strings[1:])
       except KeyError:
         pass
   else:
     for a in jsondict:
       if strings[0] in a:
         try:
           return findinjson(a[strings[0]], *strings[1:])
         except (TypeError, KeyError):  #TypeError because a could be a string and contain strings[0]
           pass
   #if it's not found
   raise KeyError("Can't find " + strings[0])
 

def dataset.getDatasetFromCache ( cachename )

Definition at line 416 of file dataset.py.

References join().

Referenced by createDataset().

 def getDatasetFromCache( cachename ) :
     cachedir =  '/'.join( [os.environ['HOME'],'.cmgdataset'])
     pckfile = open( cachedir + "/" + cachename )
     dataset = pickle.load(pckfile)      
     return dataset
 

def dataset.getrunnumbersfromfile	(	filename,
		trydas = `True`,
		allowunknown = `False`,
		dasinstance = `defaultdasinstance`
	)

Definition at line 59 of file dataset.py.

References Vispa.Plugins.EdmBrowser.EdmDataAccessor.all(), dasquery(), findinjson(), int, join(), and str.

 def getrunnumbersfromfile(filename, trydas=True, allowunknown=False, dasinstance=defaultdasinstance):
   parts = filename.split("/")
   error = None
   if parts[0] != "" or parts[1] != "store":
     error = "does not start with /store"
   elif parts[2] in ["mc", "relval"]:
     return [1]
   elif not parts[-1].endswith(".root"):
     error = "does not end with something.root"
   elif len(parts) != 12:
     error = "should be exactly 11 slashes counting the first one"
   else:
     runnumberparts = parts[-5:-2]
     if not all(len(part)==3 for part in runnumberparts):
       error = "the 3 directories {} do not have length 3 each".format("/".join(runnumberparts))
     try:
       return [int("".join(runnumberparts))]
     except ValueError:
       error = "the 3 directories {} do not form an integer".format("/".join(runnumberparts))
 
   if error and trydas:
     try:
       query = "run file={} instance={}".format(filename, dasinstance)
       dasoutput = dasquery(query)
       result = findinjson(dasoutput, "run")
       return sum((findinjson(run, "run_number") for run in result), [])
     except Exception as e:
       error = str(e)
 
   if error and allowunknown:
     return [-1]
 
   if error:
     error = "could not figure out which run number this file is from.\nMaybe try with allowunknown=True?\n  {}\n{}".format(filename, error)
     raise DatasetError(error)
 

def dataset.writeDatasetToCache	(	cachename,
		dataset
	)

Definition at line 422 of file dataset.py.

References createDataset(), and join().

Referenced by createDataset().

 def writeDatasetToCache( cachename, dataset ):
     cachedir =  '/'.join( [os.environ['HOME'],'.cmgdataset'])
     if not os.path.exists(cachedir):
         os.mkdir(cachedir)
     pckfile = open( cachedir + "/" + cachename, 'w')
     pickle.dump(dataset, pckfile)
 

Variable Documentation

dataset.abspath

Definition at line 55 of file dataset.py.

dataset.action

Definition at line 20 of file dataset.py.

dataset.args

Definition at line 38 of file dataset.py.

dataset.data

Definition at line 49 of file dataset.py.

dataset.dataset = Dataset( datasetName )

Definition at line 934 of file dataset.py.

string dataset.datasetName = '/MinimumBias/Run2012D-TkAlMinBias-v1/ALCARECO'

Definition at line 930 of file dataset.py.

Referenced by GeneralHLTOffline.dqmBeginRun().

dataset.default

Definition at line 16 of file dataset.py.

string dataset.defaultdasinstance = "prod/global"

Definition at line 15 of file dataset.py.

dataset.dest

Definition at line 16 of file dataset.py.

dataset.end

Definition at line 941 of file dataset.py.

dataset.firstRun

Definition at line 940 of file dataset.py.

Referenced by MillePedeAlignmentAlgorithm.beginRun(), SiStripLorentzAngleCalibration.beginRun(), SiPixelLorentzAngleCalibration.beginRun(), TkModuleGroupSelector.createModuleGroups(), SiPixelDcsInfo.dqmEndLuminosityBlock(), identity(), edm::SetRunHelper.setForcedRunOffset(), and SiPixelDcsInfo.SiPixelDcsInfo().

dataset.help

Definition at line 16 of file dataset.py.

dataset.info

Definition at line 46 of file dataset.py.

dataset.int

Definition at line 35 of file dataset.py.

Referenced by dataset.Dataset.__createSnippet(), dataset.Dataset.__datetime(), dataset.Dataset.__lumiSelectionSnippet(), dataset.CMSDataset.buildListOfFiles(), dataset.Dataset.convertTimeToRun(), dataset.CMSDataset.findPrimaryDatasetEntries(), dataset.PrivateDataset.findPrimaryDatasetEntries(), dataset.CMSDataset.findPrimaryDatasetNumFiles(), dataset.PrivateDataset.findPrimaryDatasetNumFiles(), dataset.Dataset.forcerunrange(), dataset.Dataset.getfiles(), dataset.Dataset.getPrimaryDatasetEntries(), dataset.Dataset.getrunnumberfromfilename(), getrunnumbersfromfile(), and dataset.BaseDataset.listOfGoodFilesWithPrescale().

tuple dataset.jsonFile

Initial value:

 = ( '/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/'
                  'Collisions12/8TeV/Prompt/'
                  'Cert_190456-207898_8TeV_PromptReco_Collisions12_JSON.txt' )

Definition at line 931 of file dataset.py.

dataset.jsonPath

Definition at line 939 of file dataset.py.

dataset.name

Definition at line 45 of file dataset.py.

dataset.options

Definition at line 38 of file dataset.py.

dataset.outName

Definition at line 938 of file dataset.py.

Referenced by SiStripCorrelateNoise.DoPlots(), TMultiDimFet.MakeCode(), ZdcTBAnalysis.setup(), and JetMETDQMPostProcessor.~JetMETDQMPostProcessor().

dataset.parser

Definition at line 14 of file dataset.py.

dataset.run_range

Definition at line 48 of file dataset.py.

dataset.type

Definition at line 35 of file dataset.py.

dataset.usage

Definition at line 15 of file dataset.py.

dataset.user

Definition at line 44 of file dataset.py.

Referenced by EcalTPGParamReaderFromDB.analyze(), EcalTPGParamReaderFromDB.EcalTPGParamReaderFromDB(), StoreESCondition.fillHeader(), StoreEcalCondition.fillHeader(), amc::Header.Header(), InputTagDistributorService.init(), VariableHelperService.init(), amc13::Packet.Packet(), amc::Packet.Packet(), VariableHelperService.set(), and SiStripConfigDb.usingDatabase().

string dataset.validationfooter

Initial value:

 = """
 ] )
 """

Definition at line 278 of file dataset.py.

string dataset.validationheader

Initial value:

 = """
 import FWCore.ParameterSet.Config as cms
 
 maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )
 readFiles = cms.untracked.vstring()
 secFiles = cms.untracked.vstring()
 source = cms.Source ("PoolSource",fileNames = readFiles, secondaryFileNames = secFiles)
 readFiles.extend( [
 """

Definition at line 268 of file dataset.py.

Classes

Functions

Variables

Function Documentation

Variable Documentation