Reference Manual

Go to the documentation of this file.
00001 #include "IOPool/Output/interface/PoolOutputModule.h"
00002 
00003 #include "FWCore/MessageLogger/interface/JobReport.h"
00004 #include "IOPool/Output/src/RootOutputFile.h"
00005 
00006 #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h"
00007 #include "FWCore/Framework/interface/RunPrincipal.h"
00008 #include "FWCore/Framework/interface/FileBlock.h"
00009 #include "FWCore/ParameterSet/interface/ParameterSet.h"
00010 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
00011 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
00012 #include "FWCore/ServiceRegistry/interface/Service.h"
00013 #include "DataFormats/Provenance/interface/BranchDescription.h"
00014 #include "FWCore/Utilities/interface/Algorithms.h"
00015 #include "FWCore/Utilities/interface/EDMException.h"
00016 #include "FWCore/Utilities/interface/ReflexTools.h"
00017 #include "FWCore/Utilities/interface/TimeOfDay.h"
00018 #include "FWCore/Utilities/interface/WrappedClassName.h"
00019 
00020 #include "TTree.h"
00021 #include "TBranchElement.h"
00022 #include "TObjArray.h"
00023 #include "RVersion.h"
00024 
00025 #include <fstream>
00026 #include <iomanip>
00027 #include <sstream>
00028 
00029 namespace edm {
00030   PoolOutputModule::PoolOutputModule(ParameterSet const& pset) :
00031     OutputModule(pset),
00032     rootServiceChecker_(),
00033     auxItems_(),
00034     selectedOutputItemList_(),
00035     fileName_(pset.getUntrackedParameter<std::string>("fileName")),
00036     logicalFileName_(pset.getUntrackedParameter<std::string>("logicalFileName")),
00037     catalog_(pset.getUntrackedParameter<std::string>("catalog")),
00038     maxFileSize_(pset.getUntrackedParameter<int>("maxSize")),
00039     compressionLevel_(pset.getUntrackedParameter<int>("compressionLevel")),
00040 #if ROOT_VERSION_CODE >= ROOT_VERSION(5,30,0)
00041     compressionAlgorithm_(pset.getUntrackedParameter<std::string>("compressionAlgorithm")),
00042 #else
00043     compressionAlgorithm_("ZLIB"),
00044 #endif
00045     basketSize_(pset.getUntrackedParameter<int>("basketSize")),
00046     eventAutoFlushSize_(pset.getUntrackedParameter<int>("eventAutoFlushCompressedSize")),
00047     splitLevel_(std::min<int>(pset.getUntrackedParameter<int>("splitLevel") + 1, 99)),
00048     basketOrder_(pset.getUntrackedParameter<std::string>("sortBaskets")),
00049     treeMaxVirtualSize_(pset.getUntrackedParameter<int>("treeMaxVirtualSize")),
00050     whyNotFastClonable_(pset.getUntrackedParameter<bool>("fastCloning") ? FileBlock::CanFastClone : FileBlock::DisabledInConfigFile),
00051     dropMetaData_(DropNone),
00052     moduleLabel_(pset.getParameter<std::string>("@module_label")),
00053     initializedFromInput_(false),
00054     outputFileCount_(0),
00055     inputFileCount_(0),
00056     childIndex_(0U),
00057     numberOfDigitsInIndex_(0U),
00058     overrideInputFileSplitLevels_(pset.getUntrackedParameter<bool>("overrideInputFileSplitLevels")),
00059     rootOutputFile_(),
00060     statusFileName_() {
00061 
00062       if (pset.getUntrackedParameter<bool>("writeStatusFile")) {
00063         std::ostringstream statusfilename;
00064         statusfilename << moduleLabel_ << '_' << getpid();
00065         statusFileName_ = statusfilename.str();
00066       }
00067 
00068       std::string dropMetaData(pset.getUntrackedParameter<std::string>("dropMetaData"));
00069       if(dropMetaData.empty()) dropMetaData_ = DropNone;
00070       else if(dropMetaData == std::string("NONE")) dropMetaData_ = DropNone;
00071       else if(dropMetaData == std::string("DROPPED")) dropMetaData_ = DropDroppedPrior;
00072       else if(dropMetaData == std::string("PRIOR")) dropMetaData_ = DropPrior;
00073       else if(dropMetaData == std::string("ALL")) dropMetaData_ = DropAll;
00074       else {
00075         throw edm::Exception(errors::Configuration, "Illegal dropMetaData parameter value: ")
00076             << dropMetaData << ".\n"
00077             << "Legal values are 'NONE', 'DROPPED', 'PRIOR', and 'ALL'.\n";
00078       }
00079 
00080     if (!wantAllEvents()) {
00081       whyNotFastClonable_+= FileBlock::EventSelectionUsed;
00082     }
00083 
00084     // We don't use this next parameter, but we read it anyway because it is part
00085     // of the configuration of this module.  An external parser creates the
00086     // configuration by reading this source code.
00087     pset.getUntrackedParameterSet("dataset");
00088   }
00089 
00090   void PoolOutputModule::beginJob() {
00091     for(int i = InEvent; i < NumBranchTypes; ++i) {
00092       BranchType branchType = static_cast<BranchType>(i);
00093       Selections const& keptVector = keptProducts()[branchType];
00094       for(Selections::const_iterator it = keptVector.begin(), itEnd = keptVector.end(); it != itEnd; ++it) {
00095         BranchDescription const& prod = **it;
00096         checkDictionaries(prod.fullClassName(), true);
00097         checkDictionaries(wrappedClassName(prod.fullClassName()), true);
00098       }
00099     }
00100   }
00101 
00102   std::string const& PoolOutputModule::currentFileName() const {
00103     return rootOutputFile_->fileName();
00104   }
00105 
00106   PoolOutputModule::AuxItem::AuxItem() :
00107         basketSize_(BranchDescription::invalidBasketSize) {}
00108 
00109   PoolOutputModule::OutputItem::OutputItem() :
00110         branchDescription_(0),
00111         product_(0),
00112         splitLevel_(BranchDescription::invalidSplitLevel),
00113         basketSize_(BranchDescription::invalidBasketSize) {}
00114 
00115   PoolOutputModule::OutputItem::OutputItem(BranchDescription const* bd, int splitLevel, int basketSize) :
00116         branchDescription_(bd),
00117         product_(0),
00118         splitLevel_(splitLevel),
00119         basketSize_(basketSize) {}
00120 
00121 
00122   PoolOutputModule::OutputItem::Sorter::Sorter(TTree* tree) : treeMap_(new std::map<std::string, int>) {
00123     // Fill a map mapping branch names to an index specifying the order in the tree.
00124     if(tree != 0) {
00125       TObjArray* branches = tree->GetListOfBranches();
00126       for(int i = 0; i < branches->GetEntries(); ++i) {
00127         TBranchElement* br = (TBranchElement*)branches->At(i);
00128         treeMap_->insert(std::make_pair(std::string(br->GetName()), i));
00129       }
00130     }
00131   }
00132 
00133   bool
00134   PoolOutputModule::OutputItem::Sorter::operator()(OutputItem const& lh, OutputItem const& rh) const {
00135     // Provides a comparison for sorting branches according to the index values in treeMap_.
00136     // Branches not found are always put at the end (i.e. not found > found).
00137     if(treeMap_->empty()) return lh < rh;
00138     std::string const& lstring = lh.branchDescription_->branchName();
00139     std::string const& rstring = rh.branchDescription_->branchName();
00140     std::map<std::string, int>::const_iterator lit = treeMap_->find(lstring);
00141     std::map<std::string, int>::const_iterator rit = treeMap_->find(rstring);
00142     bool lfound = (lit != treeMap_->end());
00143     bool rfound = (rit != treeMap_->end());
00144     if(lfound && rfound) {
00145       return lit->second < rit->second;
00146     } else if(lfound) {
00147       return true;
00148     } else if(rfound) {
00149       return false;
00150     }
00151     return lh < rh;
00152   }
00153 
00154   void PoolOutputModule::fillSelectedItemList(BranchType branchType, TTree* theInputTree) {
00155 
00156     Selections const& keptVector = keptProducts()[branchType];
00157     OutputItemList&   outputItemList = selectedOutputItemList_[branchType];
00158     AuxItem&   auxItem = auxItems_[branchType];
00159 
00160     // Fill AuxItem
00161     if (theInputTree != 0 && !overrideInputFileSplitLevels_) {
00162       TBranch* auxBranch = theInputTree->GetBranch(BranchTypeToAuxiliaryBranchName(branchType).c_str());
00163       if (auxBranch) {
00164         auxItem.basketSize_ = auxBranch->GetBasketSize();
00165       } else {
00166         auxItem.basketSize_ = basketSize_;
00167       }
00168     } else {
00169       auxItem.basketSize_ = basketSize_;
00170     }
00171 
00172     // Fill outputItemList with an entry for each branch.
00173     for(Selections::const_iterator it = keptVector.begin(), itEnd = keptVector.end(); it != itEnd; ++it) {
00174       int splitLevel = BranchDescription::invalidSplitLevel;
00175       int basketSize = BranchDescription::invalidBasketSize;
00176 
00177       BranchDescription const& prod = **it;
00178       TBranch* theBranch = ((!prod.produced() && theInputTree != 0 && !overrideInputFileSplitLevels_) ? theInputTree->GetBranch(prod.branchName().c_str()) : 0);
00179 
00180       if(theBranch != 0) {
00181         splitLevel = theBranch->GetSplitLevel();
00182         basketSize = theBranch->GetBasketSize();
00183       } else {
00184         splitLevel = (prod.splitLevel() == BranchDescription::invalidSplitLevel ? splitLevel_ : prod.splitLevel());
00185         basketSize = (prod.basketSize() == BranchDescription::invalidBasketSize ? basketSize_ : prod.basketSize());
00186       }
00187       outputItemList.emplace_back(&prod, splitLevel, basketSize);
00188     }
00189 
00190     // Sort outputItemList to allow fast copying.
00191     // The branches in outputItemList must be in the same order as in the input tree, with all new branches at the end.
00192     sort_all(outputItemList, OutputItem::Sorter(theInputTree));
00193   }
00194 
00195   void PoolOutputModule::beginInputFile(FileBlock const& fb) {
00196     if(isFileOpen()) {
00197       rootOutputFile_->beginInputFile(fb, remainingEvents());
00198     }
00199   }
00200 
00201   void PoolOutputModule::openFile(FileBlock const& fb) {
00202     if(!isFileOpen()) {
00203       doOpenFile();
00204       beginInputFile(fb);
00205     }
00206   }
00207 
00208   void PoolOutputModule::respondToOpenInputFile(FileBlock const& fb) {
00209     if(!initializedFromInput_) {
00210       for(int i = InEvent; i < NumBranchTypes; ++i) {
00211         BranchType branchType = static_cast<BranchType>(i);
00212         TTree* theInputTree = (branchType == InEvent ? fb.tree() :
00213                               (branchType == InLumi ? fb.lumiTree() :
00214                                fb.runTree()));
00215         fillSelectedItemList(branchType, theInputTree);
00216       }
00217       initializedFromInput_ = true;
00218     }
00219     ++inputFileCount_;
00220     beginInputFile(fb);
00221   }
00222 
00223   void PoolOutputModule::respondToCloseInputFile(FileBlock const& fb) {
00224     if(rootOutputFile_) rootOutputFile_->respondToCloseInputFile(fb);
00225   }
00226 
00227   void PoolOutputModule::postForkReacquireResources(unsigned int iChildIndex, unsigned int iNumberOfChildren) {
00228     childIndex_ = iChildIndex;
00229     while (iNumberOfChildren != 0) {
00230       ++numberOfDigitsInIndex_;
00231       iNumberOfChildren /= 10;
00232     }
00233     if (numberOfDigitsInIndex_ == 0) {
00234       numberOfDigitsInIndex_ = 3; // Protect against zero iNumberOfChildren
00235     }
00236   }
00237 
00238   PoolOutputModule::~PoolOutputModule() {
00239   }
00240 
00241   void PoolOutputModule::write(EventPrincipal const& e) {
00242       rootOutputFile_->writeOne(e);
00243       if (!statusFileName_.empty()) {
00244         std::ofstream statusFile(statusFileName_.c_str());
00245         statusFile << e.id() << " time: " << std::setprecision(3) << TimeOfDay() << '\n';
00246         statusFile.close();
00247       }
00248   }
00249 
00250   void PoolOutputModule::writeLuminosityBlock(LuminosityBlockPrincipal const& lb) {
00251       rootOutputFile_->writeLuminosityBlock(lb);
00252       Service<JobReport> reportSvc;
00253       reportSvc->reportLumiSection(lb.id().run(), lb.id().luminosityBlock());
00254   }
00255 
00256   void PoolOutputModule::writeRun(RunPrincipal const& r) {
00257       rootOutputFile_->writeRun(r);
00258       Service<JobReport> reportSvc;
00259       reportSvc->reportRunNumber(r.run());
00260   }
00261 
00262   // At some later date, we may move functionality from finishEndFile() to here.
00263   void PoolOutputModule::startEndFile() { }
00264 
00265 
00266   void PoolOutputModule::writeFileFormatVersion() { rootOutputFile_->writeFileFormatVersion(); }
00267   void PoolOutputModule::writeFileIdentifier() { rootOutputFile_->writeFileIdentifier(); }
00268   void PoolOutputModule::writeIndexIntoFile() { rootOutputFile_->writeIndexIntoFile(); }
00269   void PoolOutputModule::writeProcessConfigurationRegistry() { rootOutputFile_->writeProcessConfigurationRegistry(); }
00270   void PoolOutputModule::writeProcessHistoryRegistry() { rootOutputFile_->writeProcessHistoryRegistry(); }
00271   void PoolOutputModule::writeParameterSetRegistry() { rootOutputFile_->writeParameterSetRegistry(); }
00272   void PoolOutputModule::writeProductDescriptionRegistry() { rootOutputFile_->writeProductDescriptionRegistry(); }
00273   void PoolOutputModule::writeParentageRegistry() { rootOutputFile_->writeParentageRegistry(); }
00274   void PoolOutputModule::writeBranchIDListRegistry() { rootOutputFile_->writeBranchIDListRegistry(); }
00275   void PoolOutputModule::writeProductDependencies() { rootOutputFile_->writeProductDependencies(); }
00276   void PoolOutputModule::finishEndFile() { rootOutputFile_->finishEndFile(); rootOutputFile_.reset(); }
00277   bool PoolOutputModule::isFileOpen() const { return rootOutputFile_.get() != 0; }
00278   bool PoolOutputModule::shouldWeCloseFile() const { return rootOutputFile_->shouldWeCloseFile(); }
00279 
00280   void PoolOutputModule::doOpenFile() {
00281       if(inputFileCount_ == 0) {
00282         throw edm::Exception(errors::LogicError)
00283           << "Attempt to open output file before input file. "
00284           << "Please report this to the core framework developers.\n";
00285       }
00286       std::string suffix(".root");
00287       std::string::size_type offset = fileName().rfind(suffix);
00288       bool ext = (offset == fileName().size() - suffix.size());
00289       if(!ext) suffix.clear();
00290       std::string fileBase(ext ? fileName().substr(0, offset) : fileName());
00291       std::ostringstream ofilename;
00292       std::ostringstream lfilename;
00293       ofilename << fileBase;
00294       lfilename << logicalFileName();
00295       if(numberOfDigitsInIndex_) {
00296         ofilename << '_' << std::setw(numberOfDigitsInIndex_) << std::setfill('0') << childIndex_;
00297         if(!logicalFileName().empty()) {
00298           lfilename << '_' << std::setw(numberOfDigitsInIndex_) << std::setfill('0') << childIndex_;
00299         }
00300       }
00301       if(outputFileCount_) {
00302         ofilename << std::setw(3) << std::setfill('0') << outputFileCount_;
00303         if(!logicalFileName().empty()) {
00304           lfilename << std::setw(3) << std::setfill('0') << outputFileCount_;
00305         }
00306       }
00307       ofilename << suffix;
00308       rootOutputFile_.reset(new RootOutputFile(this, ofilename.str(), lfilename.str()));
00309       ++outputFileCount_;
00310   }
00311 
00312   void
00313   PoolOutputModule::fillDescriptions(ConfigurationDescriptions & descriptions) {
00314     std::string defaultString;
00315     ParameterSetDescription desc;
00316     desc.setComment("Writes runs, lumis, and events into EDM/ROOT files.");
00317     desc.addUntracked<std::string>("fileName")
00318         ->setComment("Name of output file.");
00319     desc.addUntracked<std::string>("logicalFileName", defaultString)
00320         ->setComment("Passed to job report. Otherwise unused by module.");
00321     desc.addUntracked<std::string>("catalog", defaultString)
00322         ->setComment("Passed to job report. Otherwise unused by module.");
00323     desc.addUntracked<int>("maxSize", 0x7f000000)
00324         ->setComment("Maximum output file size, in kB.\n"
00325                      "If over maximum, new output file will be started at next input file transition.");
00326     desc.addUntracked<int>("compressionLevel", 7)
00327         ->setComment("ROOT compression level of output file.");
00328 #if ROOT_VERSION_CODE >= ROOT_VERSION(5,30,0)
00329     desc.addUntracked<std::string>("compressionAlgorithm", "ZLIB")
00330         ->setComment("Algorithm used to compress data in the ROOT output file, allowed values are ZLIB and LZMA");
00331 #endif
00332     desc.addUntracked<int>("basketSize", 16384)
00333         ->setComment("Default ROOT basket size in output file.");
00334     desc.addUntracked<int>("eventAutoFlushCompressedSize",-1)->setComment("Set ROOT auto flush stored data size (in bytes) for event TTree. The value sets how large the compressed buffer is allowed to get. The uncompressed buffer can be quite a bit larger than this depending on the average compression ratio. The value of -1 just uses ROOT's default value. The value of 0 turns off this feature.");
00335     desc.addUntracked<int>("splitLevel", 99)
00336         ->setComment("Default ROOT branch split level in output file.");
00337     desc.addUntracked<std::string>("sortBaskets", std::string("sortbasketsbyoffset"))
00338         ->setComment("Legal values: 'sortbasketsbyoffset', 'sortbasketsbybranch', 'sortbasketsbyentry'.\n"
00339                      "Used by ROOT when fast copying. Affects performance.");
00340     desc.addUntracked<int>("treeMaxVirtualSize", -1)
00341         ->setComment("Size of ROOT TTree TBasket cache.  Affects performance.");
00342     desc.addUntracked<bool>("fastCloning", true)
00343         ->setComment("True:  Allow fast copying, if possible.\n"
00344                      "False: Disable fast copying.");
00345     desc.addUntracked<bool>("overrideInputFileSplitLevels", false)
00346         ->setComment("False: Use branch split levels and basket sizes from input file, if possible.\n"
00347                      "True:  Always use specified or default split levels and basket sizes.");
00348     desc.addUntracked<bool>("writeStatusFile", false)
00349         ->setComment("Write a status file. Intended for use by workflow management.");
00350     desc.addUntracked<std::string>("dropMetaData", defaultString)
00351         ->setComment("Determines handling of per product per event metadata.  Options are:\n"
00352                      "'NONE':    Keep all of it.\n"
00353                      "'DROPPED': Keep it for products produced in current process and all kept products. Drop it for dropped products produced in prior processes.\n"
00354                      "'PRIOR':   Keep it for products produced in current process. Drop it for products produced in prior processes.\n"
00355                      "'ALL':     Drop all of it.");
00356     ParameterSetDescription dataSet;
00357     dataSet.setAllowAnything();
00358     desc.addUntracked<ParameterSetDescription>("dataset", dataSet)
00359      ->setComment("PSet is only used by Data Operations and not by this module.");
00360 
00361     OutputModule::fillDescription(desc);
00362 
00363     descriptions.add("edmOutput", desc);
00364   }
00365 }
/data/refman/pasoursint/CMSSW_5_3_0/src/IOPool/Output/src/PoolOutputModule.cc
