26 #include "TBranchElement.h"
27 #include "TObjArray.h"
33 #include "boost/algorithm/string.hpp"
39 rootServiceChecker_(),
41 selectedOutputItemList_(),
42 fileName_(pset.getUntrackedParameter<std::
string>(
"fileName")),
43 logicalFileName_(pset.getUntrackedParameter<std::
string>(
"logicalFileName")),
44 catalog_(pset.getUntrackedParameter<std::
string>(
"catalog")),
45 maxFileSize_(pset.getUntrackedParameter<int>(
"maxSize")),
46 compressionLevel_(pset.getUntrackedParameter<int>(
"compressionLevel")),
47 compressionAlgorithm_(pset.getUntrackedParameter<std::
string>(
"compressionAlgorithm")),
48 basketSize_(pset.getUntrackedParameter<int>(
"basketSize")),
49 eventAuxBasketSize_(pset.getUntrackedParameter<int>(
"eventAuxiliaryBasketSize")),
50 eventAutoFlushSize_(pset.getUntrackedParameter<int>(
"eventAutoFlushCompressedSize")),
51 splitLevel_(std::
min<int>(pset.getUntrackedParameter<int>(
"splitLevel") + 1, 99)),
52 basketOrder_(pset.getUntrackedParameter<std::
string>(
"sortBaskets")),
53 treeMaxVirtualSize_(pset.getUntrackedParameter<int>(
"treeMaxVirtualSize")),
54 whyNotFastClonable_(pset.getUntrackedParameter<bool>(
"fastCloning") ?
FileBlock::CanFastClone
56 dropMetaData_(DropNone),
58 initializedFromInput_(
false),
63 overrideInputFileSplitLevels_(pset.getUntrackedParameter<bool>(
"overrideInputFileSplitLevels")),
64 compactEventAuxiliary_(pset.getUntrackedParameter<bool>(
"compactEventAuxiliary")),
65 mergeJob_(pset.getUntrackedParameter<bool>(
"mergeJob")),
69 std::ostringstream statusfilename;
75 if (dropMetaData.empty())
87 << dropMetaData <<
".\n"
88 <<
"Legal values are 'NONE', 'DROPPED', 'PRIOR', and 'ALL'.\n";
98 for (
auto const&
s : specialSplit) {
100 s.getUntrackedParameter<
int>(
"splitLevel"));
111 for (
auto const& prod : reg->productList()) {
127 : branchDescription_(bd), token_(token), product_(nullptr),
splitLevel_(splitLevel),
basketSize_(basketSize) {}
131 if (tree !=
nullptr) {
132 TObjArray* branches = tree->GetListOfBranches();
133 for (
int i = 0;
i < branches->GetEntries(); ++
i) {
134 TBranchElement*
br = (TBranchElement*)branches->At(
i);
143 if (treeMap_->empty())
147 std::map<std::string, int>::const_iterator lit = treeMap_->find(lstring);
148 std::map<std::string, int>::const_iterator rit = treeMap_->find(rstring);
149 bool lfound = (lit != treeMap_->end());
150 bool rfound = (rit != treeMap_->end());
151 if (lfound && rfound) {
152 return lit->second < rit->second;
162 return std::regex_match(iBranchName, branch_);
167 boost::replace_all(tmp,
"*",
".*");
168 boost::replace_all(tmp,
"?",
".");
169 return std::regex(tmp);
197 for (
auto const& kept : keptVector) {
206 ? theInputTree->GetBranch(prod.
branchName().c_str())
209 if (theBranch !=
nullptr) {
210 splitLevel = theBranch->GetSplitLevel();
211 basketSize = theBranch->GetBasketSize();
216 splitLevel =
b.splitLevel_;
221 outputItemList.emplace_back(&prod, kept.second, splitLevel, basketSize);
234 for (
auto const& parentToChildren : branchToChildMap) {
235 for (
auto const&
child : parentToChildren.second) {
252 std::vector<std::string>
const& processesWithProcessBlockProducts =
254 unsigned int numberOfProcessesWithProcessBlockProducts = processesWithProcessBlockProducts.size();
262 TTree* theInputTree =
268 for (
unsigned int k =
InProcess;
k < numberOfTTrees; ++
k) {
298 statusFile << e.
id() <<
" time: " << std::setprecision(3) <<
TimeOfDay() <<
'\n';
363 <<
"Please report this to the core framework developers.\n";
367 bool ext = (offset ==
fileName().size() - suffix.size());
371 std::ostringstream ofilename;
372 std::ostringstream lfilename;
373 ofilename << fileBase;
384 return std::make_pair(ofilename.str(), lfilename.str());
399 if (provenance !=
nullptr) {
402 it =
branchParents_.insert(std::make_pair(branchID, std::set<ParentageID>())).first;
427 auto pr = ep->productProvenanceRetrieverPtr();
429 pr->readProvenanceAsync(iTask, &iModuleCallingContext);
438 std::set<ParentageID>
const& eIds = branchParent.second;
439 for (
auto const& eId : eIds) {
443 for (
auto const&
parent : parents) {
453 desc.
setComment(
"Writes runs, lumis, and events into EDM/ROOT files.");
456 ->setComment(
"Passed to job report. Otherwise unused by module.");
458 ->setComment(
"Passed to job report. Otherwise unused by module.");
461 "Maximum output file size, in kB.\n"
462 "If over maximum, new output file will be started at next input file transition.");
463 desc.
addUntracked<
int>(
"compressionLevel", 4)->setComment(
"ROOT compression level of output file.");
466 "Algorithm used to compress data in the ROOT output file, allowed values are ZLIB, LZMA, LZ4, and ZSTD");
467 desc.
addUntracked<
int>(
"basketSize", 16384)->setComment(
"Default ROOT basket size in output file.");
468 desc.
addUntracked<
int>(
"eventAuxiliaryBasketSize", 16384)
469 ->setComment(
"Default ROOT basket size in output file for EventAuxiliary branch.");
470 desc.
addUntracked<
int>(
"eventAutoFlushCompressedSize", 20 * 1024 * 1024)
472 "Set ROOT auto flush stored data size (in bytes) for event TTree. The value sets how large the compressed "
473 "buffer is allowed to get. The uncompressed buffer can be quite a bit larger than this depending on the "
474 "average compression ratio. The value of -1 just uses ROOT's default value. The value of 0 turns off this "
476 desc.
addUntracked<
int>(
"splitLevel", 99)->setComment(
"Default ROOT branch split level in output file.");
479 "Legal values: 'sortbasketsbyoffset', 'sortbasketsbybranch', 'sortbasketsbyentry'.\n"
480 "Used by ROOT when fast copying. Affects performance.");
482 ->setComment(
"Size of ROOT TTree TBasket cache. Affects performance.");
485 "True: Allow fast copying, if possible.\n"
486 "False: Disable fast copying.");
489 "If set to true and fast copying is disabled, copy input file compression and basket sizes to the output "
493 "False: Write EventAuxiliary as we go like any other event metadata branch.\n"
494 "True: Optimize the file layout by deferring writing the EventAuxiliary branch until the output file is "
496 desc.
addUntracked<
bool>(
"overrideInputFileSplitLevels",
false)
498 "False: Use branch split levels and basket sizes from input file, if possible.\n"
499 "True: Always use specified or default split levels and basket sizes.");
501 ->setComment(
"Write a status file. Intended for use by workflow management.");
504 "Determines handling of per product per event metadata. Options are:\n"
505 "'NONE': Keep all of it.\n"
506 "'DROPPED': Keep it for products produced in current process and all kept products. Drop it for dropped "
507 "products produced in prior processes.\n"
508 "'PRIOR': Keep it for products produced in current process. Drop it for products produced in prior "
510 "'ALL': Drop all of it.");
515 ->setComment(
"PSet is only used by Data Operations and not by this module.");
520 "Name of branch needing a special split level. The name can contain wildcards '*' and '?'");
521 specialSplit.
addUntracked<
int>(
"splitLevel")->setComment(
"The special split level for the branch");
522 desc.
addVPSetUntracked(
"overrideBranchesSplitLevel", specialSplit, std::vector<ParameterSet>());
524 OutputModule::fillDescription(desc);
530 descriptions.
add(
"edmOutput", desc);
void writeParentageRegistry()
void openFile(FileBlock const &fb) override
list processes
Run mode ##.
void setComment(std::string const &value)
virtual std::pair< std::string, std::string > physicalAndLogicalNameForNewFile()
T getUntrackedParameter(std::string const &, T const &) const
std::string const & branchName() const
SubProcessParentageHelper const * subProcessParentageHelper() const
ProductProvenance const * branchIDToProvenanceForProducedOnly(BranchID const &bid) const
std::string const & BranchTypeToAuxiliaryBranchName(BranchType const &branchType)
BranchDescription const * branchDescription_
EventID const & id() const
BranchType const & branchType() const
void write(EventForOutput const &e) override
OutputProcessBlockHelper const & outputProcessBlockHelper() const
ParameterDescriptionBase * addUntracked(U const &iLabel, T const &value)
edm::propagate_const< std::unique_ptr< RootOutputFile > > rootOutputFile_
static int const invalidSplitLevel
void setAllowAnything()
allow any parameter label/value pairs
std::vector< SpecialSplitLevelForBranch > specialSplitLevelForBranches_
static int const invalidBasketSize
TTree * processBlockTree(std::string const &processName) const
void setProcessesWithSelectedMergeableRunProducts(std::set< std::string > const &) override
void updateBranchParents(EventForOutput const &e)
constexpr unsigned int numberOfRunLumiEventProductTrees
DropMetaData const & dropMetaData() const
BranchChildren branchChildren_
void writeRun(RunForOutput const &) override
std::string const & fileName() const
bool initializedFromInput_
void insertChild(BranchID parent, BranchID child)
BranchChildren const & branchChildren() const
std::string const moduleLabel_
std::string const & processName() const
BranchID branchID() const
ParameterSet getUntrackedParameterSet(std::string const &name, ParameterSet const &defaultValue) const
virtual void doExtrasAfterCloseFile()
std::string const & logicalFileName() const
void writeParameterSetRegistry()
std::vector< std::pair< BranchDescription const *, EDGetToken > > SelectedProducts
std::vector< std::string > const & processesWithProcessBlockProducts() const
void writeProcessBlockHelper()
std::vector< BranchID > const & parents() const
std::regex convert(std::string const &iGlobBranchExpression) const
bool wantAllEvents() const
int remainingEvents() const
PoolOutputModule(ParameterSet const &ps)
void setComment(std::string const &value)
void updateBranchParentsForOneBranch(ProductProvenanceRetriever const *provRetriever, BranchID const &branchID)
bool overrideInputFileSplitLevels_
bool operator()(OutputItem const &lh, OutputItem const &rh) const
void preActionBeforeRunEventAsync(WaitingTaskHolder iTask, ModuleCallingContext const &iModuleCallingContext, Principal const &iPrincipal) const override
std::string const & currentFileName() const
bool getMapped(key_type const &k, value_type &result) const
void writeFileFormatVersion()
map_t const & childLookup() const
std::vector< BranchID > producedBranches_
std::vector< OutputItemList > selectedOutputItemList_
OutputItem(BranchDescription const *bd, EDGetToken const &token, int splitLevel, int basketSize)
ProductProvenanceRetriever const * productProvenanceRetrieverPtr() const
void writeStoredMergeableRunProductMetadata()
void writeEventAuxiliary()
std::vector< std::string > names
BranchID const & branchID() const
EventID const & min(EventID const &lh, EventID const &rh)
void writeProductDependencies()
BranchParents branchParents_
void writeIndexIntoFile()
void writeThinnedAssociationsHelper()
void sort_all(RandomAccessSequence &s)
wrappers for std::sort
std::string const & processName() const
void writeBranchIDListRegistry()
void writeProcessHistoryRegistry()
void fillDependencyGraph()
bool shouldWeCloseFile() const override
allow inheriting classes to override but still be able to call this method in the overridden version ...
void respondToCloseInputFile(FileBlock const &fb) override
DropMetaData dropMetaData_
bool isFileOpen() const override
void reallyCloseFile() override
void add(std::string const &label, ParameterSetDescription const &psetDescription)
void writeFileIdentifier()
ParentageID const & parentageID() const
std::shared_ptr< std::map< std::string, int > > treeMap_
VParameterSet getUntrackedParameterSetVector(std::string const &name, VParameterSet const &defaultValue) const
std::string statusFileName_
SelectedProductsForBranchType const & keptProducts() const
~PoolOutputModule() override
void beginInputFile(FileBlock const &fb)
void writeProductDescriptionRegistry()
int const eventAuxBasketSize_
std::vector< OutputItem > OutputItemList
void respondToOpenInputFile(FileBlock const &fb) override
void writeProcessBlock(ProcessBlockForOutput const &) override
void fillSelectedItemList(BranchType branchtype, std::string const &processName, TTree *theInputTree, OutputItemList &)
static void fillDescriptions(ConfigurationDescriptions &descriptions)
bool match(std::string const &iBranchName) const
static void fillDescription(ParameterSetDescription &desc)
static ParentageRegistry * instance()
ParameterDescriptionBase * addVPSetUntracked(U const &iLabel, ParameterSetDescription const &validator, std::vector< ParameterSet > const &defaults)
std::vector< std::string > processesWithSelectedMergeableRunProducts_
void writeLuminosityBlock(LuminosityBlockForOutput const &) override