|
|
Go to the documentation of this file.
26 #include "TBranchElement.h"
27 #include "TObjArray.h"
33 #include "boost/algorithm/string.hpp"
39 rootServiceChecker_(),
41 selectedOutputItemList_(),
42 fileName_(
pset.getUntrackedParameter<
std::
string>(
"fileName")),
43 logicalFileName_(
pset.getUntrackedParameter<
std::
string>(
"logicalFileName")),
45 maxFileSize_(
pset.getUntrackedParameter<
int>(
"maxSize")),
46 compressionLevel_(
pset.getUntrackedParameter<
int>(
"compressionLevel")),
47 compressionAlgorithm_(
pset.getUntrackedParameter<
std::
string>(
"compressionAlgorithm")),
48 basketSize_(
pset.getUntrackedParameter<
int>(
"basketSize")),
49 eventAuxBasketSize_(
pset.getUntrackedParameter<
int>(
"eventAuxiliaryBasketSize")),
50 eventAutoFlushSize_(
pset.getUntrackedParameter<
int>(
"eventAutoFlushCompressedSize")),
51 splitLevel_(
std::
min<
int>(
pset.getUntrackedParameter<
int>(
"splitLevel") + 1, 99)),
52 basketOrder_(
pset.getUntrackedParameter<
std::
string>(
"sortBaskets")),
53 treeMaxVirtualSize_(
pset.getUntrackedParameter<
int>(
"treeMaxVirtualSize")),
54 whyNotFastClonable_(
pset.getUntrackedParameter<
bool>(
"fastCloning") ?
FileBlock::CanFastClone
56 dropMetaData_(DropNone),
57 moduleLabel_(
pset.getParameter<
std::
string>(
"@module_label")),
58 initializedFromInput_(
false),
63 overrideInputFileSplitLevels_(
pset.getUntrackedParameter<
bool>(
"overrideInputFileSplitLevels")),
64 compactEventAuxiliary_(
pset.getUntrackedParameter<
bool>(
"compactEventAuxiliary")),
67 if (
pset.getUntrackedParameter<
bool>(
"writeStatusFile")) {
68 std::ostringstream statusfilename;
87 <<
"Legal values are 'NONE', 'DROPPED', 'PRIOR', and 'ALL'.\n";
94 auto const& specialSplit{
pset.getUntrackedParameterSetVector(
"overrideBranchesSplitLevel")};
97 for (
auto const&
s : specialSplit) {
99 s.getUntrackedParameter<
int>(
"splitLevel"));
105 pset.getUntrackedParameterSet(
"dataset");
110 for (
auto const&
prod : reg->productList()) {
130 if (
tree !=
nullptr) {
131 TObjArray* branches =
tree->GetListOfBranches();
132 for (
int i = 0;
i < branches->GetEntries(); ++
i) {
133 TBranchElement*
br = (TBranchElement*)branches->At(
i);
142 if (treeMap_->empty())
144 std::string const& lstring =
lh.branchDescription_->branchName();
146 std::map<std::string, int>::const_iterator lit = treeMap_->find(lstring);
147 std::map<std::string, int>::const_iterator rit = treeMap_->find(rstring);
148 bool lfound = (lit != treeMap_->end());
149 bool rfound = (rit != treeMap_->end());
150 if (lfound && rfound) {
151 return lit->second < rit->second;
161 return std::regex_match(iBranchName, branch_);
166 boost::replace_all(
tmp,
"*",
".*");
167 boost::replace_all(
tmp,
"?",
".");
168 return std::regex(
tmp);
196 for (
auto const& kept : keptVector) {
205 ? theInputTree->GetBranch(
prod.branchName().c_str())
208 if (theBranch !=
nullptr) {
214 if (
b.match(
prod.branchName())) {
233 for (
auto const& parentToChildren : branchToChildMap) {
234 for (
auto const&
child : parentToChildren.second) {
251 std::vector<std::string>
const& processesWithProcessBlockProducts =
253 unsigned int numberOfProcessesWithProcessBlockProducts = processesWithProcessBlockProducts.size();
258 BranchType branchType = static_cast<BranchType>(
i);
261 TTree* theInputTree =
267 for (
unsigned int k =
InProcess;
k < numberOfTTrees; ++
k) {
297 statusFile <<
e.id() <<
" time: " << std::setprecision(3) <<
TimeOfDay() <<
'\n';
362 <<
"Please report this to the core framework developers.\n";
370 std::ostringstream ofilename;
371 std::ostringstream lfilename;
372 ofilename << fileBase;
383 return std::make_pair(ofilename.str(), lfilename.str());
398 if (provenance !=
nullptr) {
424 auto const*
ep = dynamic_cast<EventPrincipal const*>(&iPrincipal);
426 auto pr =
ep->productProvenanceRetrieverPtr();
428 pr->readProvenanceAsync(iTask, &iModuleCallingContext);
437 std::set<ParentageID>
const& eIds = branchParent.second;
438 for (
auto const& eId : eIds) {
452 desc.setComment(
"Writes runs, lumis, and events into EDM/ROOT files.");
453 desc.addUntracked<
std::string>(
"fileName")->setComment(
"Name of output file.");
455 ->setComment(
"Passed to job report. Otherwise unused by module.");
457 ->setComment(
"Passed to job report. Otherwise unused by module.");
458 desc.addUntracked<
int>(
"maxSize", 0x7f000000)
460 "Maximum output file size, in kB.\n"
461 "If over maximum, new output file will be started at next input file transition.");
462 desc.addUntracked<
int>(
"compressionLevel", 9)->setComment(
"ROOT compression level of output file.");
465 "Algorithm used to compress data in the ROOT output file, allowed values are ZLIB, LZMA, and ZSTD");
466 desc.addUntracked<
int>(
"basketSize", 16384)->setComment(
"Default ROOT basket size in output file.");
467 desc.addUntracked<
int>(
"eventAuxiliaryBasketSize", 16384)
468 ->setComment(
"Default ROOT basket size in output file for EventAuxiliary branch.");
469 desc.addUntracked<
int>(
"eventAutoFlushCompressedSize", 20 * 1024 * 1024)
471 "Set ROOT auto flush stored data size (in bytes) for event TTree. The value sets how large the compressed "
472 "buffer is allowed to get. The uncompressed buffer can be quite a bit larger than this depending on the "
473 "average compression ratio. The value of -1 just uses ROOT's default value. The value of 0 turns off this "
475 desc.addUntracked<
int>(
"splitLevel", 99)->setComment(
"Default ROOT branch split level in output file.");
478 "Legal values: 'sortbasketsbyoffset', 'sortbasketsbybranch', 'sortbasketsbyentry'.\n"
479 "Used by ROOT when fast copying. Affects performance.");
480 desc.addUntracked<
int>(
"treeMaxVirtualSize", -1)
481 ->setComment(
"Size of ROOT TTree TBasket cache. Affects performance.");
482 desc.addUntracked<
bool>(
"fastCloning",
true)
484 "True: Allow fast copying, if possible.\n"
485 "False: Disable fast copying.");
486 desc.addUntracked<
bool>(
"compactEventAuxiliary",
false)
488 "False: Write EventAuxiliary as we go like any other event metadata branch.\n"
489 "True: Optimize the file layout be deferring writing the EventAuxiliary branch until the output file is "
491 desc.addUntracked<
bool>(
"overrideInputFileSplitLevels",
false)
493 "False: Use branch split levels and basket sizes from input file, if possible.\n"
494 "True: Always use specified or default split levels and basket sizes.");
495 desc.addUntracked<
bool>(
"writeStatusFile",
false)
496 ->setComment(
"Write a status file. Intended for use by workflow management.");
499 "Determines handling of per product per event metadata. Options are:\n"
500 "'NONE': Keep all of it.\n"
501 "'DROPPED': Keep it for products produced in current process and all kept products. Drop it for dropped "
502 "products produced in prior processes.\n"
503 "'PRIOR': Keep it for products produced in current process. Drop it for products produced in prior "
505 "'ALL': Drop all of it.");
510 ->setComment(
"PSet is only used by Data Operations and not by this module.");
515 "Name of branch needing a special split level. The name can contain wildcards '*' and '?'");
516 specialSplit.
addUntracked<
int>(
"splitLevel")->setComment(
"The special split level for the branch");
517 desc.addVPSetUntracked(
"overrideBranchesSplitLevel", specialSplit, std::vector<ParameterSet>());
519 OutputModule::fillDescription(
desc);
525 descriptions.
add(
"edmOutput",
desc);
~PoolOutputModule() override
int remainingEvents() const
std::vector< BranchID > const & parents() const
void updateBranchParents(EventForOutput const &e)
BranchChildren branchChildren_
void respondToOpenInputFile(FileBlock const &fb) override
void writeProcessBlock(ProcessBlockForOutput const &) override
void writeLuminosityBlock(LuminosityBlockForOutput const &) override
SubProcessParentageHelper const * subProcessParentageHelper() const
BranchID branchID() const
std::regex convert(std::string const &iGlobBranchExpression) const
std::vector< std::string > processesWithSelectedMergeableRunProducts_
void sort_all(RandomAccessSequence &s)
wrappers for std::sort
EventID const & min(EventID const &lh, EventID const &rh)
edm::propagate_const< std::unique_ptr< RootOutputFile > > rootOutputFile_
void openFile(FileBlock const &fb) override
void writeParameterSetRegistry()
static ParentageRegistry * instance()
std::string const & currentFileName() const
bool initializedFromInput_
void writeProcessBlockHelper()
bool match(std::string const &iBranchName) const
void setProcessesWithSelectedMergeableRunProducts(std::set< std::string > const &) override
void updateBranchParentsForOneBranch(ProductProvenanceRetriever const *provRetriever, BranchID const &branchID)
BranchDescription const * branchDescription_
virtual void doExtrasAfterCloseFile()
std::string const & BranchTypeToAuxiliaryBranchName(BranchType const &branchType)
std::vector< OutputItemList > selectedOutputItemList_
std::vector< SpecialSplitLevelForBranch > specialSplitLevelForBranches_
std::vector< std::pair< BranchDescription const *, EDGetToken > > SelectedProducts
PoolOutputModule(ParameterSet const &ps)
constexpr unsigned int numberOfRunLumiEventProductTrees
void writeRun(RunForOutput const &) override
void preActionBeforeRunEventAsync(WaitingTaskHolder iTask, ModuleCallingContext const &iModuleCallingContext, Principal const &iPrincipal) const override
const std::string moduleLabel_
void add(std::string const &label, ParameterSetDescription const &psetDescription)
void writeFileFormatVersion()
const std::string names[nVars_]
bool operator()(OutputItem const &lh, OutputItem const &rh) const
std::vector< BranchID > producedBranches_
BranchParents branchParents_
static const int invalidSplitLevel
DropMetaData const & dropMetaData() const
OutputItem(BranchDescription const *bd, EDGetToken const &token, int splitLevel, int basketSize)
bool wantAllEvents() const
std::shared_ptr< std::map< std::string, int > > treeMap_
TTree * processBlockTree(std::string const &processName) const
bool overrideInputFileSplitLevels_
void writeThinnedAssociationsHelper()
void writeEventAuxiliary()
ParameterDescriptionBase * addUntracked(U const &iLabel, T const &value)
void writeIndexIntoFile()
bool getMapped(key_type const &k, value_type &result) const
ProductProvenance const * branchIDToProvenanceForProducedOnly(BranchID const &bid) const
SelectedProductsForBranchType const & keptProducts() const
void writeProductDependencies()
map_t const & childLookup() const
std::string const & branchName() const
ParentageID const & parentageID() const
bool isFileOpen() const override
void writeStoredMergeableRunProductMetadata()
std::string const & processName() const
void beginInputFile(FileBlock const &fb)
std::string const & fileName() const
void writeProductDescriptionRegistry()
void writeBranchIDListRegistry()
BranchChildren const & branchChildren() const
void fillDependencyGraph()
static void fillDescriptions(ConfigurationDescriptions &descriptions)
bool shouldWeCloseFile() const override
allow inheriting classes to override but still be able to call this method in the overridden version
void reallyCloseFile() override
std::vector< std::string > const & processesWithProcessBlockProducts() const
std::string statusFileName_
std::string const & logicalFileName() const
static const int invalidBasketSize
OutputProcessBlockHelper const & outputProcessBlockHelper() const
void writeProcessHistoryRegistry()
void respondToCloseInputFile(FileBlock const &fb) override
DropMetaData dropMetaData_
void writeFileIdentifier()
std::vector< OutputItem > OutputItemList
void write(EventForOutput const &e) override
void fillSelectedItemList(BranchType branchtype, std::string const &processName, TTree *theInputTree, OutputItemList &)
void writeParentageRegistry()
static void fillDescription(ParameterSetDescription &desc)
void insertChild(BranchID parent, BranchID child)
virtual std::pair< std::string, std::string > physicalAndLogicalNameForNewFile()
const int eventAuxBasketSize_