dataset Namespace Reference

Classes
class	BaseDataset
class	CMSDataset
class	DataFile
class	Dataset
class	DatasetBase
class	DatasetError
class	EOSDataset
class	IntegrityCheckError
class	LocalDataset
class	MultipleDatasets
class	PrivateDataset
class	RunRange

Functions
def	createDataset (user, dataset, pattern, readcache=False, basedir=None, run_range=None)
def	createMyDataset (user, dataset, pattern, dbsInstance, readcache=False)
	if user == 'CMS': data = CMSDataset( dataset ) elif user == 'LOCAL': if basedir is None: basedir = os.environ['CMGLOCALBASEDIR'] data = LocalDataset( dataset, basedir, pattern ) else: data = Dataset( user, dataset, pattern ) More...
def	dasquery (dasQuery, dasLimit=0)
def	findinjson (jsondict, strings)
def	getDatasetFromCache (cachename)
def	getrunnumbersfromfile (filename, trydas=True, allowunknown=False, dasinstance=defaultdasinstance)
def	writeDatasetToCache (cachename, dataset)

Variables
	abspath
	action
	args
	data
	default
string	defaultdasinstance = "prod/global"
	dest
	help
	info
	int
	name
	options
	parser
	run_range
	type
	usage
	user
string	validationfooter
string	validationheader

Function Documentation

createDataset()

def dataset.createDataset	(		user,
			dataset,
			pattern,
			readcache = False,
			basedir = None,
			run_range = None
	)

Definition at line 429 of file dataset.py.

References getDatasetFromCache(), and writeDatasetToCache().

Referenced by datasetToSource.datasetToSource(), production_tasks.CheckDatasetExists.run(), production_tasks.SourceCFG.run(), and writeDatasetToCache().

                   basedir = None, run_range = None):
    
    
    def cacheFileName(data, user, pattern):
        return '{user}%{name}%{pattern}.pck'.format( user = user, name = data.replace('/','_'), pattern = pattern)

    def writeCache(dataset):
        writeDatasetToCache( cacheFileName(dataset.name, dataset.user, dataset.pattern), dataset )

    def readCache(data, user, pattern):
        return getDatasetFromCache( cacheFileName(data, user, pattern) )

    if readcache:
        try:
            data = readCache(dataset, user, pattern)
        except IOError:
            readcache = False
    if not readcache:
        if user == 'CMS':
            data = CMSDataset( dataset , run_range = run_range)
            info = False
        elif user == 'LOCAL':
            data = LocalDataset( dataset, basedir, pattern)
            info = False
        elif user == 'EOS':
            data = EOSDataset(dataset, basedir, pattern)
            info = False
        else:
            data = Dataset( dataset, user, pattern)
        writeCache(data)

createMyDataset()

def dataset.createMyDataset	(		user,
			dataset,
			pattern,
			dbsInstance,
			readcache = False
	)

if user == 'CMS': data = CMSDataset( dataset ) elif user == 'LOCAL': if basedir is None: basedir = os.environ['CMGLOCALBASEDIR'] data = LocalDataset( dataset, basedir, pattern ) else: data = Dataset( user, dataset, pattern )

MM

Definition at line 470 of file dataset.py.

References join().

Referenced by datasetToSource.myDatasetToSource().

def createMyDataset( user, dataset, pattern, dbsInstance, readcache=False):

    cachedir =  '/'.join( [os.environ['HOME'],'.cmgdataset'])

    def cacheFileName(data, user, dbsInstance, pattern):
        cf =  data.replace('/','_')
        name = '{dir}/{user}%{dbsInstance}%{name}%{pattern}.pck'.format(
            dir = cachedir,
            user = user,
            dbsInstance = dbsInstance,
            name = cf,
            pattern = pattern)
        return name

    def writeCache(dataset):
        if not os.path.exists(cachedir):
            os.mkdir(cachedir)
        cachename = cacheFileName(dataset.name,
                                  dataset.user,
                                  dataset.dbsInstance,
                                  dataset.pattern)
        pckfile = open( cachename, 'w')
        pickle.dump(dataset, pckfile)

    def readCache(data, user, dbsInstance, pattern):
        cachename = cacheFileName(data, user, dbsInstance, pattern)
        
        pckfile = open( cachename)
        dataset = pickle.load(pckfile)
        #print 'reading cache'                                                                                                                                                                   
        return dataset

    if readcache:
        try:
            data = readCache(dataset, user, dbsInstance, pattern)    
        except IOError:
            readcache = False
    if not readcache:
        if user == 'PRIVATE':
            data = PrivateDataset( dataset, dbsInstance )
            info = False
        writeCache(data)
    return data

dasquery()

def dataset.dasquery	(		dasQuery,
			dasLimit = 0
	)

Definition at line 27 of file dataset.py.

References findinjson(), das_client.get_data(), and str.

Referenced by dataset.Dataset.getfiles(), and getrunnumbersfromfile().

def dasquery(dasQuery, dasLimit=0):
  dasData = das_client.get_data(dasQuery, dasLimit)
  if isinstance(dasData, str):
    jsondict = json.loads( dasData )
  else:
    jsondict = dasData
  # Check, if the DAS query fails
  try:
    error = findinjson(jsondict, "data","error")
  except KeyError:
    error = None
  if error or findinjson(jsondict, "status") != 'ok' or "data" not in jsondict:
    try:
      jsonstr = findinjson(jsondict, "reason")
    except KeyError: 
      jsonstr = str(jsondict)
    if len(jsonstr) > 10000:
      jsonfile = "das_query_output_%i.txt"
      i = 0
      while os.path.lexists(jsonfile % i):
        i += 1
      jsonfile = jsonfile % i
      theFile = open( jsonfile, "w" )
      theFile.write( jsonstr )
      theFile.close()
      msg = "The DAS query returned an error.  The output is very long, and has been stored in:\n" + jsonfile
    else:
      msg = "The DAS query returned a error.  Here is the output\n" + jsonstr
    msg += "\nIt's possible that this was a server error.  If so, it may work if you try again later"
    raise DatasetError(msg)
  return findinjson(jsondict, "data")

findinjson()

def dataset.findinjson	(		jsondict,
			strings
	)

Definition at line 95 of file dataset.py.

Referenced by dasquery(), dataset.Dataset.getfiles(), and getrunnumbersfromfile().

def findinjson(jsondict, *strings):
  if len(strings) == 0:
    return jsondict
  if isinstance(jsondict,dict):
    if strings[0] in jsondict:
      try:
        return findinjson(jsondict[strings[0]], *strings[1:])
      except KeyError:
        pass
  else:
    for a in jsondict:
      if strings[0] in a:
        try:
          return findinjson(a[strings[0]], *strings[1:])
        except (TypeError, KeyError):  #TypeError because a could be a string and contain strings[0]
          pass
  #if it's not found
  raise KeyError("Can't find " + strings[0])

getDatasetFromCache()

def dataset.getDatasetFromCache

(

cachename

)

Definition at line 415 of file dataset.py.

References join().

Referenced by createDataset().

def getDatasetFromCache( cachename ) :
    cachedir =  '/'.join( [os.environ['HOME'],'.cmgdataset'])
    pckfile = open( cachedir + "/" + cachename )
    dataset = pickle.load(pckfile)      
    return dataset

getrunnumbersfromfile()

def dataset.getrunnumbersfromfile	(		filename,
			trydas = True,
			allowunknown = False,
			dasinstance = defaultdasinstance
	)

Definition at line 59 of file dataset.py.

References python.cmstools.all(), dasquery(), findinjson(), int, join(), and str.

def getrunnumbersfromfile(filename, trydas=True, allowunknown=False, dasinstance=defaultdasinstance):
  parts = filename.split("/")
  error = None
  if parts[0] != "" or parts[1] != "store":
    error = "does not start with /store"
  elif parts[2] in ["mc", "relval"]:
    return [1]
  elif not parts[-1].endswith(".root"):
    error = "does not end with something.root"
  elif len(parts) != 12:
    error = "should be exactly 11 slashes counting the first one"
  else:
    runnumberparts = parts[-5:-2]
    if not all(len(part)==3 for part in runnumberparts):
      error = "the 3 directories {} do not have length 3 each".format("/".join(runnumberparts))
    try:
      return [int("".join(runnumberparts))]
    except ValueError:
      error = "the 3 directories {} do not form an integer".format("/".join(runnumberparts))

  if error and trydas:
    try:
      query = "run file={} instance={}".format(filename, dasinstance)
      dasoutput = dasquery(query)
      result = findinjson(dasoutput, "run")
      return sum((findinjson(run, "run_number") for run in result), [])
    except Exception as e:
      error = str(e)

  if error and allowunknown:
    return [-1]

  if error:
    error = "could not figure out which run number this file is from.\nMaybe try with allowunknown=True?\n  {}\n{}".format(filename, error)
    raise DatasetError(error)

writeDatasetToCache()

def dataset.writeDatasetToCache	(		cachename,
			dataset
	)

Definition at line 421 of file dataset.py.

References createDataset(), and join().

Referenced by createDataset().

def writeDatasetToCache( cachename, dataset ):
    cachedir =  '/'.join( [os.environ['HOME'],'.cmgdataset'])
    if not os.path.exists(cachedir):
        os.mkdir(cachedir)
    pckfile = open( cachedir + "/" + cachename, 'w')
    pickle.dump(dataset, pckfile)

Variable Documentation

abspath

dataset.abspath

Definition at line 55 of file dataset.py.

action

dataset.action

Definition at line 20 of file dataset.py.

args

dataset.args

Definition at line 38 of file dataset.py.

data

dataset.data

Definition at line 49 of file dataset.py.

default

dataset.default

Definition at line 16 of file dataset.py.

defaultdasinstance

string dataset.defaultdasinstance = "prod/global"

Definition at line 15 of file dataset.py.

dest

dataset.dest

Definition at line 16 of file dataset.py.

help

dataset.help

Definition at line 16 of file dataset.py.

info

dataset.info

Definition at line 46 of file dataset.py.

int

dataset.int

Definition at line 35 of file dataset.py.

Referenced by dataset.CMSDataset.buildListOfFiles(), dataset.CMSDataset.findPrimaryDatasetEntries(), dataset.PrivateDataset.findPrimaryDatasetEntries(), dataset.CMSDataset.findPrimaryDatasetNumFiles(), dataset.PrivateDataset.findPrimaryDatasetNumFiles(), dataset.Dataset.getfiles(), dataset.Dataset.getPrimaryDatasetEntries(), getrunnumbersfromfile(), and dataset.BaseDataset.listOfGoodFilesWithPrescale().

name

dataset.name

Definition at line 45 of file dataset.py.

options

dataset.options

Definition at line 38 of file dataset.py.

parser

dataset.parser

Definition at line 14 of file dataset.py.

run_range

dataset.run_range

Definition at line 48 of file dataset.py.

type

dataset.type

Definition at line 35 of file dataset.py.

usage

dataset.usage

Definition at line 15 of file dataset.py.

user

dataset.user

Definition at line 44 of file dataset.py.

validationfooter

string dataset.validationfooter

Initial value:

=  """
] )
"""

Definition at line 278 of file dataset.py.

validationheader

string dataset.validationheader

Initial value:

=  """
import FWCore.ParameterSet.Config as cms

maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )
readFiles = cms.untracked.vstring()
secFiles = cms.untracked.vstring()
source = cms.Source ("PoolSource",fileNames = readFiles, secondaryFileNames = secFiles)
readFiles.extend( [
"""

Definition at line 268 of file dataset.py.