CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Static Public Member Functions | Public Attributes
dataset.CMSDataset Class Reference
Inheritance diagram for dataset.CMSDataset:
dataset.BaseDataset

Public Member Functions

def __init__
 
def buildListOfFiles
 
def buildListOfFilesDBS
 
def getPrimaryDatasetEntries
 
- Public Member Functions inherited from dataset.BaseDataset
def __init__
 def init(self, name, user, pattern='. More...
 
def buildListOfBadFiles
 
def buildListOfFiles
 
def extractFileSizes
 
def getPrimaryDatasetEntries
 
def listOfFiles
 
def listOfGoodFiles
 
def listOfGoodFilesWithPrescale
 
def printFiles
 
def printInfo
 

Static Public Member Functions

def findPrimaryDatasetEntries
 
def findPrimaryDatasetNumFiles
 

Public Attributes

 files
 
- Public Attributes inherited from dataset.BaseDataset
 bad_files
 
 dbsInstance
 MM. More...
 
 files
 
 filesAndSizes
 
 good_files
 
 name
 
 pattern
 
 primaryDatasetEntries
 MM. More...
 
 report
 
 run_range
 
 user
 

Detailed Description

Definition at line 126 of file dataset.py.

Constructor & Destructor Documentation

def dataset.CMSDataset.__init__ (   self,
  name,
  run_range = None 
)

Definition at line 128 of file dataset.py.

128 
129  def __init__(self, name, run_range = None):
130  super(CMSDataset, self).__init__( name, 'CMS', run_range=run_range)

Member Function Documentation

def dataset.CMSDataset.buildListOfFiles (   self,
  pattern = '.*root' 
)

Definition at line 160 of file dataset.py.

References dataset.CMSDataset.findPrimaryDatasetNumFiles(), and dataset.BaseDataset.run_range.

160 
161  def buildListOfFiles(self, pattern='.*root'):
162  runs = (-1,-1)
163  if self.run_range is not None:
164  runs = self.run_range
165  num_files=self.findPrimaryDatasetNumFiles(self.name.rstrip('/'),
166  runs[0],runs[1])
167  limit = 10000
168  if num_files > limit:
169  num_steps = int(num_files/limit)+1
170  self.files = []
171  for i in xrange(num_steps):
172  DBSFiles=self.buildListOfFilesDBS(pattern,
173  i*limit,
174  ((i+1)*limit)-1)
175  self.files.extend(DBSFiles)
176  else:
177  self.files = self.buildListOfFilesDBS(pattern)
dataset.CMSDataset.buildListOfFilesDBS
def buildListOfFilesDBS
Definition: dataset.py:131
dataset.CMSDataset.findPrimaryDatasetNumFiles
def findPrimaryDatasetNumFiles
Definition: dataset.py:202
def dataset.CMSDataset.buildListOfFilesDBS (   self,
  pattern,
  begin = -1,
  end = -1 
)

Definition at line 131 of file dataset.py.

References dataset.BaseDataset.run_range.

131 
132  def buildListOfFilesDBS(self, pattern, begin=-1, end=-1):
133  print 'buildListOfFilesDBS',begin,end
134  sampleName = self.name.rstrip('/')
135  query, qwhat = sampleName, "dataset"
136  if "#" in sampleName: qwhat = "block"
137  if self.run_range is not None and self.run_range != (-1,-1):
138  if self.run_range[0] == self.run_range[1]:
139  query += " run=%s" % self.run_range[0]
140  else:
141  print "WARNING: queries with run ranges are slow in DAS"
142  query += " run between [%s,%s]" % ( self.run_range[0],self.run_range[1] )
143  dbs='das_client.py --query="file %s=%s"'%(qwhat,query)
144  if begin >= 0:
145  dbs += ' --index %d' % begin
146  if end >= 0:
147  dbs += ' --limit %d' % (end-begin+1)
148  else:
149  dbs += ' --limit 0'
150  print 'dbs\t: %s' % dbs
151  dbsOut = os.popen(dbs)
152  files = []
153  for line in dbsOut:
154  if line.find('/store')==-1:
155  continue
156  line = line.rstrip()
157  # print 'line',line
158  files.append(line)
159  return files
dataset.CMSDataset.buildListOfFilesDBS
def buildListOfFilesDBS
Definition: dataset.py:131
def dataset.CMSDataset.findPrimaryDatasetEntries (   dataset,
  runmin,
  runmax 
)
static

Definition at line 179 of file dataset.py.

Referenced by dataset.CMSDataset.getPrimaryDatasetEntries(), and dataset.PrivateDataset.getPrimaryDatasetEntries().

179 
180  def findPrimaryDatasetEntries(dataset, runmin, runmax):
181 
182  query, qwhat = dataset, "dataset"
183  if "#" in dataset: qwhat = "block"
184  if runmin >0 or runmax > 0:
185  if runmin == runmax:
186  query = "%s run=%d" % (query,runmin)
187  else:
188  print "WARNING: queries with run ranges are slow in DAS"
189  query = "%s run between [%d, %d]" % (query,runmin if runmin > 0 else 1, runmax if runmax > 0 else 999999)
190  dbs='das_client.py --query="summary %s=%s"'%(qwhat,query)
191  dbsOut = os.popen(dbs).readlines()
192 
193  entries = []
194  for line in dbsOut:
195  line = line.replace('\n','')
196  if "nevents" in line:
197  entries.append(int(line.split(":")[1]))
198  if entries:
199  return sum(entries)
200  return -1
dataset.CMSDataset.findPrimaryDatasetEntries
def findPrimaryDatasetEntries
Definition: dataset.py:179
def dataset.CMSDataset.findPrimaryDatasetNumFiles (   dataset,
  runmin,
  runmax 
)
static

Definition at line 202 of file dataset.py.

Referenced by dataset.CMSDataset.buildListOfFiles(), and dataset.PrivateDataset.buildListOfFilesDBS().

202 
203  def findPrimaryDatasetNumFiles(dataset, runmin, runmax):
204 
205  query, qwhat = dataset, "dataset"
206  if "#" in dataset: qwhat = "block"
207  if runmin >0 or runmax > 0:
208  if runmin == runmax:
209  query = "%s run=%d" % (query,runmin)
210  else:
211  print "WARNING: queries with run ranges are slow in DAS"
212  query = "%s run between [%d, %d]" % (query,runmin if runmin > 0 else 1, runmax if runmax > 0 else 999999)
213  dbs='das_client.py --query="summary %s=%s"'%(qwhat,query)
214  dbsOut = os.popen(dbs).readlines()
215 
216  entries = []
217  for line in dbsOut:
218  line = line.replace('\n','')
219  if "nfiles" in line:
220  entries.append(int(line.split(":")[1]))
221  if entries:
222  return sum(entries)
223  return -1
dataset.CMSDataset.findPrimaryDatasetNumFiles
def findPrimaryDatasetNumFiles
Definition: dataset.py:202
def dataset.CMSDataset.getPrimaryDatasetEntries (   self)

Definition at line 224 of file dataset.py.

References dataset.CMSDataset.findPrimaryDatasetEntries(), ElectronMVAID.ElectronMVAID.name, counter.Counter.name, entry.name, average.Average.name, histograms.Histograms.name, TmModule.name, cond::persistency::TAG::NAME.name, cond::persistency::GLOBAL_TAG::NAME.name, core.autovars.NTupleVariable.name, cond::persistency::TAG::TIME_TYPE.name, cond::persistency::GLOBAL_TAG::VALIDITY.name, genericValidation.GenericValidation.name, cond::persistency::COND_LOG_TABLE::EXECTIME.name, preexistingValidation.PreexistingValidation.name, cond::persistency::TAG::OBJECT_TYPE.name, cond::persistency::GLOBAL_TAG::DESCRIPTION.name, cond::persistency::COND_LOG_TABLE::IOVTAG.name, cond::persistency::TAG::SYNCHRONIZATION.name, cond::persistency::GLOBAL_TAG::RELEASE.name, ora::RecordSpecImpl::Item.name, cond::persistency::COND_LOG_TABLE::USERTEXT.name, cond::persistency::TAG::END_OF_VALIDITY.name, cond::persistency::GLOBAL_TAG::SNAPSHOT_TIME.name, cond::persistency::TAG::DESCRIPTION.name, cond::persistency::GTEditorData.name, cond::persistency::GLOBAL_TAG::INSERTION_TIME.name, cond::persistency::TAG::LAST_VALIDATED_TIME.name, FWTGeoRecoGeometry::Info.name, Types._Untracked.name, cond::persistency::TAG::INSERTION_TIME.name, cond::persistency::TAG::MODIFICATION_TIME.name, dataset.BaseDataset.name, personalPlayback.Applet.name, ParameterSet.name, PixelDCSObject< class >::Item.name, analyzer.Analyzer.name, DQMRivetClient::LumiOption.name, MagCylinder.name, alignment.Alignment.name, ParSet.name, DQMRivetClient::ScaleFactorOption.name, SingleObjectCondition.name, EgHLTOfflineSummaryClient::SumHistBinData.name, XMLRBXPedestalsLoader::_loaderBaseConfig.name, XMLHTRZeroSuppressionLoader::_loaderBaseConfig.name, DQMGenericClient::EfficOption.name, cond::persistency::GTProxyData.name, core.autovars.NTupleObjectType.name, MyWatcher.name, edm::PathTimingSummary.name, cond::TimeTypeSpecs.name, lumi::TriggerInfo.name, edm::PathSummary.name, cond::persistency::GLOBAL_TAG_MAP::GLOBAL_TAG_NAME.name, perftools::EdmEventSize::BranchRecord.name, PixelEndcapLinkMaker::Item.name, FWTableViewManager::TableEntry.name, RecAnalyzerMinbias.name, cond::persistency::GLOBAL_TAG_MAP::RECORD.name, PixelBarrelLinkMaker::Item.name, Mapper::definition< ScannerT >.name, EcalLogicID.name, cond::persistency::GLOBAL_TAG_MAP::LABEL.name, cond::persistency::GLOBAL_TAG_MAP::TAG_NAME.name, ExpressionHisto< T >.name, McSelector.name, RecoSelector.name, XMLProcessor::_loaderBaseConfig.name, DQMGenericClient::ProfileOption.name, cond::persistency::PAYLOAD::HASH.name, TreeCrawler.Package.name, cond::persistency::PAYLOAD::OBJECT_TYPE.name, cond::persistency::PAYLOAD::DATA.name, cond::persistency::PAYLOAD::STREAMER_INFO.name, cond::persistency::PAYLOAD::VERSION.name, MagGeoBuilderFromDDD::volumeHandle.name, cond::persistency::PAYLOAD::INSERTION_TIME.name, DQMGenericClient::NormOption.name, options.ConnectionHLTMenu.name, DQMGenericClient::CDOption.name, h4DSegm.name, cond::TagInfo_t.name, PhysicsTools::Calibration::Variable.name, looper.Looper.name, EDMtoMEConverter.name, MEtoEDM< T >::MEtoEDMObject.name, cond::persistency::IOV::TAG_NAME.name, cond::persistency::IOV::SINCE.name, cond::persistency::IOV::PAYLOAD_HASH.name, cond::persistency::IOV::INSERTION_TIME.name, MuonGeometrySanityCheckPoint.name, config.Analyzer.name, config.Service.name, core.autovars.NTupleSubObject.name, core.autovars.NTupleObject.name, h2DSegm.name, options.HLTProcessOptions.name, DQMNet::WaitObject.name, AlpgenParameterName.name, SiStripMonitorDigi.name, core.autovars.NTupleCollection.name, cond::persistency::TAG_MIGRATION::SOURCE_ACCOUNT.name, cond::persistency::TAG_MIGRATION::SOURCE_TAG.name, cond::persistency::TAG_MIGRATION::TAG_NAME.name, cond::persistency::TAG_MIGRATION::STATUS_CODE.name, cond::persistency::TAG_MIGRATION::INSERTION_TIME.name, FastTimerService::LuminosityDescription.name, cond::persistency::PAYLOAD_MIGRATION::SOURCE_ACCOUNT.name, cond::persistency::PAYLOAD_MIGRATION::SOURCE_TOKEN.name, cond::persistency::PAYLOAD_MIGRATION::PAYLOAD_HASH.name, cond::persistency::PAYLOAD_MIGRATION::INSERTION_TIME.name, conddblib.Tag.name, conddblib.GlobalTag.name, personalPlayback.FrameworkJob.name, plotscripts.SawTeethFunction.name, FastTimerService::ProcessDescription.name, hTMaxCell.name, cscdqm::ParHistoDef.name, BeautifulSoup.Tag.name, TiXmlAttribute.name, BeautifulSoup.SoupStrainer.name, and dataset.BaseDataset.run_range.

224 
225  def getPrimaryDatasetEntries(self):
226  runmin = -1
227  runmax = -1
228  if self.run_range is not None:
229  runmin = self.run_range[0]
230  runmax = self.run_range[1]
231  return self.findPrimaryDatasetEntries(self.name, runmin, runmax)
dataset.CMSDataset.getPrimaryDatasetEntries
def getPrimaryDatasetEntries
Definition: dataset.py:224
dataset.CMSDataset.findPrimaryDatasetEntries
def findPrimaryDatasetEntries
Definition: dataset.py:179

Member Data Documentation

dataset.CMSDataset.files

Definition at line 169 of file dataset.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5