#include <Example/ExternalLHEProducer/src/ExternalLHEProducer.cc>

Inheritance diagram for ExternalLHEProducer:

Classes
class	FileCloseSentry

Public Member Functions
	ExternalLHEProducer (const edm::ParameterSet &iConfig)

	~ExternalLHEProducer () override

Public Member Functions inherited from edm::one::EDProducer< edm::BeginRunProducer, edm::EndRunProducer >
	EDProducer ()=default

Public Member Functions inherited from edm::one::EDProducerBase
	EDProducerBase ()

ModuleDescription const &	moduleDescription () const

	~EDProducerBase () override

Public Member Functions inherited from edm::ProducerBase
void	callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)

std::vector< edm::ProductResolverIndex > const &	indiciesForPutProducts (BranchType iBranchType) const

	ProducerBase ()

std::vector< edm::ProductResolverIndex > const &	putTokenIndexToProductResolverIndex () const

void	registerProducts (ProducerBase , ProductRegistry , ModuleDescription const &)

std::function< void(BranchDescription const &)>	registrationCallback () const
	used by the fwk to register list of products More...

void	resolvePutIndicies (BranchType iBranchType, ModuleToResolverIndicies const &iIndicies, std::string const &moduleLabel)

virtual	~ProducerBase () noexcept(false)

Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const

void	convertCurrentProcessAlias (std::string const &processName)
	Convert "@currentProcess" in InputTag process names to the actual current process name. More...

	EDConsumerBase ()

	EDConsumerBase (EDConsumerBase const &)=delete

	EDConsumerBase (EDConsumerBase &&)=default

ProductResolverIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const

void	itemsMayGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

void	itemsToGet (BranchType, std::vector< ProductResolverIndexAndSkipBit > &) const

std::vector< ProductResolverIndexAndSkipBit > const &	itemsToGetFrom (BranchType iType) const

void	labelsForToken (EDGetToken iToken, Labels &oLabels) const

void	modulesWhoseProductsAreConsumed (std::vector< ModuleDescription const * > &modules, ProductRegistry const &preg, std::map< std::string, ModuleDescription const * > const &labelsToDesc, std::string const &processName) const

EDConsumerBase const &	operator= (EDConsumerBase const &)=delete

EDConsumerBase &	operator= (EDConsumerBase &&)=default

bool	registeredToConsume (ProductResolverIndex, bool, BranchType) const

bool	registeredToConsumeMany (TypeID const &, BranchType) const

void	updateLookup (BranchType iBranchType, ProductResolverIndexHelper const &, bool iPrefetchMayGet)

virtual	~EDConsumerBase () noexcept(false)

Static Public Member Functions
static void	fillDescriptions (edm::ConfigurationDescriptions &descriptions)

Static Public Member Functions inherited from edm::one::EDProducerBase
static const std::string &	baseType ()

static void	fillDescriptions (ConfigurationDescriptions &descriptions)

static void	prevalidate (ConfigurationDescriptions &descriptions)

Private Member Functions
void	beginRunProduce (edm::Run &run, edm::EventSetup const &es) override

int	closeDescriptors (int preserve)

void	endRunProduce (edm::Run &, edm::EventSetup const &) override

void	executeScript ()

void	nextEvent ()

void	preallocThreads (unsigned int) override

void	produce (edm::Event &, const edm::EventSetup &) override

std::unique_ptr< std::string >	readOutput ()

Private Attributes
std::vector< std::string >	args_

uint32_t	nEvents_

uint32_t	npars_

unsigned int	nThreads_ {1}

std::string	outputContents_

std::string	outputFile_

boost::shared_ptr< lhef::LHEEvent >	partonLevel

std::unique_ptr< lhef::LHEReader >	reader_

boost::shared_ptr< lhef::LHERunInfo >	runInfo

boost::shared_ptr< lhef::LHERunInfo >	runInfoLast

boost::ptr_deque< LHERunInfoProduct >	runInfoProducts

std::string	scriptName_

bool	storeXML_

bool	wasMerged

Additional Inherited Members
Public Types inherited from edm::one::EDProducerBase
typedef EDProducerBase	ModuleType

Public Types inherited from edm::ProducerBase
using	ModuleToResolverIndicies = std::unordered_multimap< std::string, std::tuple< edm::TypeID const , const char , edm::ProductResolverIndex >>

typedef ProductRegistryHelper::TypeLabelList	TypeLabelList

Public Types inherited from edm::EDConsumerBase
typedef ProductLabels	Labels

Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)

EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)

ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...

template<typename ProductType , BranchType B = InEvent>
void	consumesMany ()

void	consumesMany (const TypeToGet &id)

template<BranchType B>
void	consumesMany (const TypeToGet &id)

template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)

EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

Detailed Description

Description: [one line class summary]

Implementation: [Notes on implementation]

Definition at line 70 of file ExternalLHEProducer.cc.

Constructor & Destructor Documentation

ExternalLHEProducer::ExternalLHEProducer ( const edm::ParameterSet & iConfig )

explicit

Definition at line 133 of file ExternalLHEProducer.cc.

References args_, and npars_.

                                                                        :
   scriptName_((iConfig.getParameter<edm::FileInPath>("scriptName")).fullPath()),
   outputFile_(iConfig.getParameter<std::string>("outputFile")),
   args_(iConfig.getParameter<std::vector<std::string> >("args")),
   npars_(iConfig.getParameter<uint32_t>("numberOfParameters")),
   nEvents_(iConfig.getUntrackedParameter<uint32_t>("nEvents")),
   storeXML_(iConfig.getUntrackedParameter<bool>("storeXML"))
 {
   if (npars_ != args_.size())
     throw cms::Exception("ExternalLHEProducer") << "Problem with configuration: " << args_.size() << " script arguments given, expected " << npars_;
   produces<LHEXMLStringProduct, edm::Transition::BeginRun>("LHEScriptOutput"); 
 
   produces<LHEEventProduct>();
   produces<LHERunInfoProduct, edm::Transition::BeginRun>();
   produces<LHERunInfoProduct, edm::Transition::EndRun>();
 }

ExternalLHEProducer::~ExternalLHEProducer ( )

override

Definition at line 151 of file ExternalLHEProducer.cc.

152 {

153 }

Member Function Documentation

void ExternalLHEProducer::beginRunProduce	(	edm::Run &	run,
		edm::EventSetup const &	es
	)

overrideprivate

Definition at line 228 of file ExternalLHEProducer.cc.

References LHERunInfoProduct::addComment(), LHERunInfoProduct::addHeader(), args_, Exception, executeScript(), edm::Service< T >::isAvailable(), LogDebug, eostools::move(), edm::RandomNumberGenerator::mySeed(), nEvents_, nextEvent(), nThreads_, outputFile_, AlCaHLTBitMon_ParallelJobs::p, edm::Run::put(), reader_, runInfo, runInfoLast, runInfoProducts, createPayload::skip, storeXML_, and wasMerged.

 {
 
   // pass the number of events as previous to last argument
   
   std::ostringstream eventStream;
   eventStream << nEvents_;
   // args_.push_back(eventStream.str());
   args_.insert(args_.begin() + 1, eventStream.str());
 
   // pass the random number generator seed as last argument
 
   edm::Service<edm::RandomNumberGenerator> rng;
 
   if ( ! rng.isAvailable()) {
     throw cms::Exception("Configuration")
       << "The ExternalLHEProducer module requires the RandomNumberGeneratorService\n"
       "which is not present in the configuration file.  You must add the service\n"
       "in the configuration file if you want to run ExternalLHEProducer";
   }
   std::ostringstream randomStream;
   randomStream << rng->mySeed(); 
   // args_.push_back(randomStream.str());
   args_.insert(args_.begin() + 2, randomStream.str());
 
   // args_.emplace_back(std::to_string(nThreads_));
   args_.insert(args_.begin() + 3, std::to_string(nThreads_));
 
   for ( unsigned int iArg = 0; iArg < args_.size() ; iArg++ ) {
     LogDebug("LHEInputArgs") << "arg [" << iArg << "] = " << args_[iArg];
   }
 
   executeScript();
   
   //fill LHEXMLProduct (streaming read directly into compressed buffer to save memory)
   std::unique_ptr<LHEXMLStringProduct> p(new LHEXMLStringProduct);
 
   //store the XML file only if explictly requested
   if (storeXML_) {
     std::ifstream instream(outputFile_);
     if (!instream) {
       throw cms::Exception("OutputOpenError") << "Unable to open script output file " << outputFile_ << ".";
     }  
     instream.seekg (0, instream.end);
     int insize = instream.tellg();
     instream.seekg (0, instream.beg);  
     p->fillCompressedContent(instream, 0.25*insize);
     instream.close();
   }
   run.put(std::move(p), "LHEScriptOutput");
 
   // LHE C++ classes translation
   // (read back uncompressed file from disk in streaming mode again to save memory)
 
   std::vector<std::string> infiles(1, outputFile_);
   unsigned int skip = 0;
   reader_ = std::make_unique<lhef::LHEReader>(infiles, skip);
 
   nextEvent();
   if (runInfoLast) {
     runInfo = runInfoLast;
   
     std::unique_ptr<LHERunInfoProduct> product(new LHERunInfoProduct(*runInfo->getHEPRUP()));
     std::for_each(runInfo->getHeaders().begin(),
                   runInfo->getHeaders().end(),
                   boost::bind(&LHERunInfoProduct::addHeader,
                               product.get(), _1));
     std::for_each(runInfo->getComments().begin(),
                   runInfo->getComments().end(),
                   boost::bind(&LHERunInfoProduct::addComment,
                               product.get(), _1));
   
     // keep a copy around in case of merging
     runInfoProducts.push_back(new LHERunInfoProduct(*product));
     wasMerged = false;
   
     run.put(std::move(product));
   
     runInfo.reset();
   }
 
 }

int ExternalLHEProducer::closeDescriptors ( int preserve )

private

Definition at line 345 of file ExternalLHEProducer.cc.

References dir, and reco::dp.

Referenced by executeScript().

 {
   int maxfd = 1024;
   int fd;
 #ifdef __linux__
   DIR * dir;
   struct dirent *dp;
   maxfd = preserve;
   if ((dir = opendir("/proc/self/fd"))) {
     errno = 0;
     while ((dp = readdir (dir)) != nullptr) {
       if ((strcmp(dp->d_name, ".") == 0)  || (strcmp(dp->d_name, "..") == 0)) {
         continue;
       }
       if (sscanf(dp->d_name, "%d", &fd) != 1) {
         //throw cms::Exception("closeDescriptors") << "Found unexpected filename in /proc/self/fd: " << dp->d_name;
         return -1;
       }
       if (fd > maxfd) {
         maxfd = fd;
       }
     }
     if (errno) {
       //throw cms::Exception("closeDescriptors") << "Unable to determine the number of fd (errno=" << errno << ", " << strerror(errno) << ").";
       return errno;
     }
     closedir(dir);
   }
 #endif
   // TODO: assert for an unreasonable number of fds?
   for (fd=3; fd<maxfd+1; fd++) {
     if (fd != preserve)
       close(fd);
   }
   return 0;
 }

void ExternalLHEProducer::endRunProduce	(	edm::Run &	run,
		edm::EventSetup const &	es
	)

overrideprivate

Definition at line 313 of file ExternalLHEProducer.cc.

References Exception, eostools::move(), nextEvent(), outputFile_, partonLevel, edm::Run::put(), reader_, and runInfoProducts.

 {
 
   if (!runInfoProducts.empty()) {
     std::unique_ptr<LHERunInfoProduct> product(runInfoProducts.pop_front().release());
     run.put(std::move(product));
   }
   
   nextEvent();
   if (partonLevel) {
     // VALIDATION_RUN env variable allows to finish event processing early without errors by sending SIGINT
     if (std::getenv("VALIDATION_RUN") != nullptr) {
       edm::LogWarning("ExternalLHEProducer")
           << "Event loop is over, but there are still lhe events to process, ignoring...";
     } else {
       throw cms::Exception("ExternalLHEProducer")
           << "Error in ExternalLHEProducer::endRunProduce().  "
           << "Event loop is over, but there are still lhe events to process."
           << "This could happen if lhe file contains more events than requested.  This is never expected to happen.";
     }
   }  
   
   reader_.reset();  
   
   if (unlink(outputFile_.c_str())) {
     throw cms::Exception("OutputDeleteError") << "Unable to delete original script output file " << outputFile_ << " (errno=" << errno << ", " << strerror(errno) << ").";
   }  
 
 }

void ExternalLHEProducer::executeScript ( )

private

Definition at line 384 of file ExternalLHEProducer.cc.

References edm::TimingServiceBase::addToCPUTime(), dir2webdir::argc, args_, cmsBatch::argv, closeDescriptors(), Exception, mps_fire::i, edm::Service< T >::isAvailable(), NULL, sysUtil::pid, pipe::pipe(), scriptName_, mps_update::status, ntuplemaker::time, and TriggerAnalyzer::write().

Referenced by beginRunProduce().

 {
 
   // Fork a script, wait until it finishes.
 
   int rc = 0, rc2 = 0;
   int filedes[2], fd_flags;
   unsigned int argc;
 
   if (pipe(filedes)) {
     throw cms::Exception("Unable to create a new pipe");
   }
   FileCloseSentry sentry1(filedes[0]), sentry2(filedes[1]);
 
   if ((fd_flags = fcntl(filedes[1], F_GETFD, NULL)) == -1) {
     throw cms::Exception("ExternalLHEProducer") << "Failed to get pipe file descriptor flags (errno=" << rc << ", " << strerror(rc) << ")";
   }
   if (fcntl(filedes[1], F_SETFD, fd_flags | FD_CLOEXEC) == -1) {
     throw cms::Exception("ExternalLHEProducer") << "Failed to set pipe file descriptor flags (errno=" << rc << ", " << strerror(rc) << ")";
   }
 
   argc = 1 + args_.size();
   // TODO: assert that we have a reasonable number of arguments
   char **argv = new char *[argc+1];
   argv[0] = strdup(scriptName_.c_str());
   for (unsigned int i=1; i<argc; i++) {
     argv[i] = strdup(args_[i-1].c_str());
   }
   argv[argc] = nullptr;
 
   pid_t pid = fork();
   if (pid == 0) {
     // The child process
     if (!(rc = closeDescriptors(filedes[1]))) {
       execvp(argv[0], argv); // If execv returns, we have an error.
       rc = errno;
     }
     while ((write(filedes[1], &rc, sizeof(int)) == -1) && (errno == EINTR)) {}
     _exit(1);
   }
 
   // Free the arg vector ASAP
   for (unsigned int i=0; i<args_.size()+1; i++) {
     free(argv[i]);
   }
   delete [] argv;
 
   if (pid == -1) {
     throw cms::Exception("ForkException") << "Unable to fork a child (errno=" << errno << ", " << strerror(errno) << ")";
   }
 
   close(filedes[1]);
   // If the exec succeeds, the read will fail.
   while (((rc2 = read(filedes[0], &rc, sizeof(int))) == -1) && (errno == EINTR)) { rc2 = 0; }
   if ((rc2 == sizeof(int)) && rc) {
     throw cms::Exception("ExternalLHEProducer") << "Failed to execute script (errno=" << rc << ", " << strerror(rc) << ")";
   }
   close(filedes[0]);
 
   int status = 0;
   errno = 0;
   do {
     if (waitpid(pid, &status, 0) < 0) {
       if (errno == EINTR) {
         continue;
       } else {
         throw cms::Exception("ExternalLHEProducer") << "Failed to read child status (errno=" << errno << ", " << strerror(errno) << ")";
       }
     }
     if (WIFSIGNALED(status)) {
       throw cms::Exception("ExternalLHEProducer") << "Child exited due to signal " << WTERMSIG(status) << ".";
     }
     if (WIFEXITED(status)) {
       rc = WEXITSTATUS(status);
       break;
     }
   } while (true);
   edm::Service<edm::TimingServiceBase> ts;
   if(ts.isAvailable()) {
     struct rusage ru;
     getrusage(RUSAGE_CHILDREN,&ru);
     double time = static_cast<double>(ru.ru_stime.tv_sec) + (static_cast<double>(ru.ru_stime.tv_usec) * 1E-6);
     ts->addToCPUTime(time);
   }
   if (rc) {
     throw cms::Exception("ExternalLHEProducer") << "Child failed with exit code " << rc << ".";
   }
 
 }

void ExternalLHEProducer::fillDescriptions ( edm::ConfigurationDescriptions & descriptions )

static

Definition at line 504 of file ExternalLHEProducer.cc.

References edm::ParameterSetDescription::add(), edm::ConfigurationDescriptions::addDefault(), edm::ParameterSetDescription::addUntracked(), edm::ParameterSetDescription::setComment(), and AlCaHLTBitMon_QueryRunRegistry::string.

                                                                                 {
   //The following says we do not know what parameters are allowed so do no validation
   // Please change this to state exactly what you do use, even if it is no parameters
   edm::ParameterSetDescription desc;
   desc.setComment("Executes an external script and places its output file into an EDM collection");
 
   edm::FileInPath thePath;
   desc.add<edm::FileInPath>("scriptName", thePath);
   desc.add<std::string>("outputFile", "myoutput");
   desc.add<std::vector<std::string> >("args");
   desc.add<uint32_t>("numberOfParameters");
   desc.addUntracked<uint32_t>("nEvents");
   desc.addUntracked<bool>("storeXML", false);
 
   descriptions.addDefault(desc);
 }

void ExternalLHEProducer::nextEvent ( )

private

Definition at line 521 of file ExternalLHEProducer.cc.

References DEFINE_FWK_MODULE, partonLevel, reader_, runInfo, and runInfoLast.

Referenced by beginRunProduce(), endRunProduce(), and produce().

 {
 
   if (partonLevel)
     return;
 
   if(not reader_) { return;}
   partonLevel = reader_->next();
   if (!partonLevel)
     return;
 
   boost::shared_ptr<lhef::LHERunInfo> runInfoThis = partonLevel->getRunInfo();
   if (runInfoThis != runInfoLast) {
     runInfo = runInfoThis;
     runInfoLast = runInfoThis;
   }
 }

void ExternalLHEProducer::preallocThreads ( unsigned int iThreads )

overrideprivatevirtual

Reimplemented from edm::one::EDProducerBase.

Definition at line 162 of file ExternalLHEProducer.cc.

References nThreads_.

 {
   nThreads_ = iThreads;
 }

void ExternalLHEProducer::produce	(	edm::Event &	iEvent,
		const edm::EventSetup &	iSetup
	)

overrideprivate

Definition at line 169 of file ExternalLHEProducer.cc.

References LHEEventProduct::addComment(), LHERunInfoProduct::addComment(), LHERunInfoProduct::addHeader(), LHEEventProduct::addWeight(), Exception, eostools::move(), nextEvent(), partonLevel, edm::Event::put(), runInfo, runInfoProducts, and wasMerged.

 {
   nextEvent();
   if (!partonLevel) {
     throw cms::Exception("ExternalLHEProducer") << "No lhe event found in ExternalLHEProducer::produce().  "
     << "The likely cause is that the lhe file contains fewer events than were requested, which is possible "
     << "in case of phase space integration or uneweighting efficiency problems.";
   }
 
   std::unique_ptr<LHEEventProduct> product(
            new LHEEventProduct(*partonLevel->getHEPEUP(),
                    partonLevel->originalXWGTUP())
            );
   if (partonLevel->getPDF()) {
     product->setPDF(*partonLevel->getPDF());
   }
   std::for_each(partonLevel->weights().begin(),
                 partonLevel->weights().end(),
                 boost::bind(&LHEEventProduct::addWeight,
                             product.get(), _1));
   product->setScales(partonLevel->scales());
   product->setNpLO(partonLevel->npLO());
   product->setNpNLO(partonLevel->npNLO());
   std::for_each(partonLevel->getComments().begin(),
                 partonLevel->getComments().end(),
                 boost::bind(&LHEEventProduct::addComment,
                             product.get(), _1));
 
   iEvent.put(std::move(product));
 
   if (runInfo) {
     std::auto_ptr<LHERunInfoProduct> product(new LHERunInfoProduct(*runInfo->getHEPRUP()));
     std::for_each(runInfo->getHeaders().begin(),
                   runInfo->getHeaders().end(),
                   boost::bind(&LHERunInfoProduct::addHeader,
                               product.get(), _1));
     std::for_each(runInfo->getComments().begin(),
                   runInfo->getComments().end(),
                   boost::bind(&LHERunInfoProduct::addComment,
                               product.get(), _1));
   
     if (!runInfoProducts.empty()) {
       runInfoProducts.front().mergeProduct(*product);
       if (!wasMerged) {
         runInfoProducts.pop_front();
         runInfoProducts.push_front(product);
         wasMerged = true;
       }
     }
   
     runInfo.reset();
   }
   
   partonLevel.reset();
   return; 
 }

std::unique_ptr< std::string > ExternalLHEProducer::readOutput ( )

private

Definition at line 476 of file ExternalLHEProducer.cc.

References BUFSIZE, Exception, gen::n, outputFile_, and AlCaHLTBitMon_QueryRunRegistry::string.

 {
   int fd;
   ssize_t n;
   char buf[BUFSIZE];
 
   if ((fd = open(outputFile_.c_str(), O_RDONLY)) == -1) {
     throw cms::Exception("OutputOpenError") << "Unable to open script output file " << outputFile_ << " (errno=" << errno << ", " << strerror(errno) << ").";
   }
 
   std::stringstream ss;
   while ((n = read(fd, buf, BUFSIZE)) > 0 || (n == -1 && errno == EINTR)) {
     if (n > 0)
       ss.write(buf, n);
   }
   if (n == -1) {
     throw cms::Exception("OutputOpenError") << "Unable to read from script output file " << outputFile_ << " (errno=" << errno << ", " << strerror(errno) << ").";
   }
 
   if (unlink(outputFile_.c_str())) {
     throw cms::Exception("OutputDeleteError") << "Unable to delete original script output file " << outputFile_ << " (errno=" << errno << ", " << strerror(errno) << ").";
   }
 
   return std::unique_ptr<std::string>(new std::string(ss.str()));
 }