#include <StreamerInputSource.h>

Inheritance diagram for edm::StreamerInputSource:

Classes
class	ProductGetter
Public Member Functions
void	deserializeAndMergeWithRegistry (InitMsgView const &initView, bool subsequent=false)
EventPrincipal *	deserializeEvent (EventMsgView const &eventView)
	StreamerInputSource (ParameterSet const &pset, InputSourceDescription const &desc)
virtual	~StreamerInputSource ()
Static Public Member Functions
static std::auto_ptr < SendJobHeader >	deserializeRegistry (InitMsgView const &initView)
static void	fillDescription (ParameterSetDescription &description)
static void	mergeIntoRegistry (SendJobHeader const &header, ProductRegistry &, bool subsequent)
static unsigned int	uncompressBuffer (unsigned char *inputBuffer, unsigned int inputSize, std::vector< unsigned char > &outputBuffer, unsigned int expectedFullSize)
Protected Member Functions
void	resetAfterEndRun ()
void	setEndRun ()
Static Protected Member Functions
static void	buildClassCache (SendDescs const &descs)
static void	declareStreamers (SendDescs const &descs)
Protected Attributes
bool	inputFileTransitionsEachEvent_
Private Member Functions
virtual ItemType	getNextItemType ()
virtual EventPrincipal *	read ()=0
virtual EventPrincipal *	readEvent_ ()
virtual boost::shared_ptr < FileBlock >	readFile_ ()
virtual boost::shared_ptr < LuminosityBlockAuxiliary >	readLuminosityBlockAuxiliary_ ()
virtual boost::shared_ptr < RunAuxiliary >	readRunAuxiliary_ ()
virtual void	setRun (RunNumber_t r)
Private Attributes
std::vector< unsigned char >	dest_
bool	eventCached_
bool	newLumi_
bool	newRun_
ProductGetter	productGetter_
bool	runEndingFlag_
TClass *	tc_
TBufferFile	xbuf_
Static Private Attributes
static std::string	processName_
static unsigned int	protocolVersion_

Detailed Description

Definition at line 29 of file StreamerInputSource.h.

Constructor & Destructor Documentation

edm::StreamerInputSource::StreamerInputSource	(	ParameterSet const &	pset,
		InputSourceDescription const &	desc
	)		`[explicit]`

Definition at line 49 of file StreamerInputSource.cc.

                                                       :
    InputSource(pset, desc),
    // The default value for the following parameter get defined in at least one derived class
    // where it has a different default value.
    inputFileTransitionsEachEvent_(
      pset.getUntrackedParameter<bool>("inputFileTransitionsEachEvent", false)),
    newRun_(true),
    newLumi_(true),
    eventCached_(false),
    tc_(getTClass(typeid(SendEvent))),
    dest_(init_size),
    xbuf_(TBuffer::kRead, init_size),
    runEndingFlag_(false),
    productGetter_() {
  }

edm::StreamerInputSource::~StreamerInputSource ( ) [virtual]

Definition at line 67 of file StreamerInputSource.cc.

{}

Member Function Documentation

void edm::StreamerInputSource::buildClassCache ( SendDescs const & descs ) [static, protected]

Definition at line 115 of file StreamerInputSource.cc.

References edm::doBuildRealData(), FDEBUG, i, and edm::wrappedClassName().

Referenced by mergeIntoRegistry().

                                                             {
    SendDescs::const_iterator i(descs.begin()), e(descs.end());

    for(; i != e; ++i) {
        //pi->init();
        std::string const real_name = wrappedClassName(i->className());
        FDEBUG(6) << "BuildReadData: " << real_name << std::endl;
        doBuildRealData(real_name);
    }
  }

void edm::StreamerInputSource::declareStreamers ( SendDescs const & descs ) [static, protected]

Definition at line 102 of file StreamerInputSource.cc.

References FDEBUG, i, edm::loadCap(), and edm::wrappedClassName().

Referenced by mergeIntoRegistry().

                                                              {
    SendDescs::const_iterator i(descs.begin()), e(descs.end());

    for(; i != e; ++i) {
        //pi->init();
        std::string const real_name = wrappedClassName(i->className());
        FDEBUG(6) << "declare: " << real_name << std::endl;
        loadCap(real_name);
    }
  }

void edm::StreamerInputSource::deserializeAndMergeWithRegistry	(	InitMsgView const &	initView,
		bool	subsequent = `false`
	)

Deserializes the specified init message into a SendJobHeader object and merges registries.

Definition at line 240 of file StreamerInputSource.cc.

References edm::PrincipalCache::adjustEventToNewProductRegistry(), deserializeRegistry(), i, edm::detail::ThreadSafeRegistry< KEY, T, E >::insertMapped(), edm::detail::ThreadSafeRegistry< KEY, T, E >::instance(), mergeIntoRegistry(), edm::InputSource::principalCache(), edm::InputSource::productRegistry(), edm::InputSource::productRegistryUpdate(), and MultipleCompare::pset.

Referenced by edm::StreamerFileReader::read(), edm::EventStreamHttpReader::readHeader(), edm::StreamerFileReader::StreamerFileReader(), and edm::StreamerInputModule< Producer >::StreamerInputModule().

                                                                                                   {
     std::auto_ptr<SendJobHeader> sd = deserializeRegistry(initView);
     ProcessConfigurationVector const& pcv = sd->processConfigurations();
     mergeIntoRegistry(*sd, productRegistryUpdate(), subsequent);
     if (subsequent) {
       principalCache().adjustEventToNewProductRegistry(productRegistry());
     }
     SendJobHeader::ParameterSetMap const& psetMap = sd->processParameterSet();
     pset::Registry& psetRegistry = *pset::Registry::instance();
     for (SendJobHeader::ParameterSetMap::const_iterator i = psetMap.begin(), iEnd = psetMap.end(); i != iEnd; ++i) {
       ParameterSet pset(i->second.pset());
       pset.setID(i->first);
       psetRegistry.insertMapped(pset);
     }
     ProcessConfigurationRegistry& pcReg = *ProcessConfigurationRegistry::instance();
     for (ProcessConfigurationVector::const_iterator it = pcv.begin(), itEnd = pcv.end(); it != itEnd; ++it) {
       pcReg.insertMapped(*it);
     }
  }

EventPrincipal * edm::StreamerInputSource::deserializeEvent ( EventMsgView const & eventView )

Deserializes the specified event message into an EventPrincipal object.

Definition at line 264 of file StreamerInputSource.cc.

Referenced by edm::StreamerFileReader::read(), and edm::EventStreamHttpReader::read().

                                                                     {
    if(eventView.code() != Header::EVENT)
      throw cms::Exception("StreamTranslation","Event deserialization error")
        << "received wrong message type: expected EVENT, got "
        << eventView.code() << "\n";
    FDEBUG(9) << "Decode event: "
         << eventView.event() << " "
         << eventView.run() << " "
         << eventView.size() << " "
         << eventView.adler32_chksum() << " "
         << eventView.eventLength() << " "
         << eventView.eventData()
         << std::endl;
    EventSourceSentry(*this);
    // uncompress if we need to
    // 78 was a dummy value (for no uncompressed) - should be 0 for uncompressed
    // need to get rid of this when 090 MTCC streamers are gotten rid of
    unsigned long origsize = eventView.origDataSize();
    unsigned long dest_size; //(should be >= eventView.origDataSize())

    uint32_t adler32_chksum = cms::Adler32((char*)eventView.eventData(), eventView.eventLength());
    //std::cout << "Adler32 checksum of event = " << adler32_chksum << std::endl;
    //std::cout << "Adler32 checksum from header = " << eventView.adler32_chksum() << " "
    //          << "host name = " << eventView.hostName() << " len = " << eventView.hostName_len() << std::endl;
    if((uint32)adler32_chksum != eventView.adler32_chksum()) {
      std::cerr << "Error from StreamerInputSource: checksum of event data blob failed "
                << " chksum from event = " << adler32_chksum << " from header = "
                << eventView.adler32_chksum() << " host name = " << eventView.hostName() << std::endl;
      // skip event (based on option?) or throw exception?
    }
    if(origsize != 78 && origsize != 0) {
      // compressed
      dest_size = uncompressBuffer((unsigned char*)eventView.eventData(),
                                   eventView.eventLength(), dest_, origsize);
    } else { // not compressed
      // we need to copy anyway the buffer as we are using dest in xbuf
      dest_size = eventView.eventLength();
      dest_.resize(dest_size);
      unsigned char* pos = (unsigned char*) &dest_[0];
      unsigned char* from = (unsigned char*) eventView.eventData();
      std::copy(from,from+dest_size,pos);
    }
    //TBuffer xbuf(TBuffer::kRead, dest_size,
    //             (char*) &dest[0],kFALSE);
    //TBuffer xbuf(TBuffer::kRead, eventView.eventLength(),
    //             (char*) eventView.eventData(),kFALSE);
    xbuf_.Reset();
    xbuf_.SetBuffer(&dest_[0],dest_size,kFALSE);
    RootDebug tracer(10,10);

    setRefCoreStreamer(&productGetter_);
    std::auto_ptr<SendEvent> sd((SendEvent*)xbuf_.ReadObjectAny(tc_));
    setRefCoreStreamer();

    if(sd.get()==0) {
        throw cms::Exception("StreamTranslation","Event deserialization error")
          << "got a null event from input stream\n";
    }
    ProcessHistoryRegistry::instance()->insertMapped(sd->processHistory());

    FDEBUG(5) << "Got event: " << sd->aux().id() << " " << sd->products().size() << std::endl;
    if(runAuxiliary().get() == 0 || runAuxiliary()->run() != sd->aux().run()) {
      newRun_ = newLumi_ = true;
      RunAuxiliary* runAuxiliary = new RunAuxiliary(sd->aux().run(), sd->aux().time(), Timestamp::invalidTimestamp());
      runAuxiliary->setProcessHistoryID(sd->processHistory().id());
      setRunAuxiliary(runAuxiliary);
      readAndCacheRun();
      setRunPrematurelyRead();
    }
    if(!luminosityBlockAuxiliary() || luminosityBlockAuxiliary()->luminosityBlock() != eventView.lumi()) {
      LuminosityBlockAuxiliary* luminosityBlockAuxiliary =
        new LuminosityBlockAuxiliary(runAuxiliary()->run(), eventView.lumi(), sd->aux().time(), Timestamp::invalidTimestamp());
      luminosityBlockAuxiliary->setProcessHistoryID(sd->processHistory().id());
      setLuminosityBlockAuxiliary(luminosityBlockAuxiliary);
      newLumi_ = true;
      readAndCacheLumi();
      setLumiPrematurelyRead();
    }

    boost::shared_ptr<EventSelectionIDVector> ids(new EventSelectionIDVector(sd->eventSelectionIDs()));
    boost::shared_ptr<BranchListIndexes> indexes(new BranchListIndexes(sd->branchListIndexes()));
    std::auto_ptr<EventAuxiliary> aux(new EventAuxiliary(sd->aux()));
    eventPrincipalCache()->fillEventPrincipal(aux, luminosityBlockPrincipal(), ids, indexes);
    productGetter_.setEventPrincipal(eventPrincipalCache());
    eventCached_ = true;

    // no process name list handling

    SendProds & sps = sd->products();
    for(SendProds::iterator spi = sps.begin(), spe = sps.end(); spi != spe; ++spi) {
        FDEBUG(10) << "check prodpair" << std::endl;
        if(spi->desc() == 0)
          throw cms::Exception("StreamTranslation","Empty Provenance");
        FDEBUG(5) << "Prov:"
             << " " << spi->desc()->className()
             << " " << spi->desc()->productInstanceName()
             << " " << spi->desc()->branchID()
             << std::endl;

        ConstBranchDescription branchDesc(*spi->desc());
        // This ProductProvenance constructor inserts into the entry description registry
        std::auto_ptr<ProductProvenance> productProvenance(
             new ProductProvenance(spi->branchID(),
                                   spi->status(),
                                   *spi->parents()));

        if(spi->prod() != 0) {
          std::auto_ptr<EDProduct> aprod(const_cast<EDProduct*>(spi->prod()));
          FDEBUG(10) << "addgroup next " << spi->branchID() << std::endl;
          eventPrincipalCache()->putOnRead(branchDesc, aprod, productProvenance);
          FDEBUG(10) << "addgroup done" << std::endl;
        } else {
          FDEBUG(10) << "addgroup empty next " << spi->branchID() << std::endl;
          eventPrincipalCache()->putOnRead(branchDesc, std::auto_ptr<EDProduct>(), productProvenance);
          FDEBUG(10) << "addgroup empty done" << std::endl;
        }
        spi->clear();
    }

    FDEBUG(10) << "Size = " << eventPrincipalCache()->size() << std::endl;

    return eventPrincipalCache();
  }

std::auto_ptr< SendJobHeader > edm::StreamerInputSource::deserializeRegistry ( InitMsgView const & initView ) [static]

Deserializes the specified init message into a SendJobHeader object (which is related to the product registry).

Definition at line 192 of file StreamerInputSource.cc.

References cms::Adler32(), InitMsgView::adler32_chksum(), benchmark_cfg::cerr, InitMsgView::code(), InitMsgView::descData(), InitMsgView::descLength(), Exception, FDEBUG, edm::getTClass(), InitMsgView::hostName(), Header::INIT, InitMsgView::processName(), processName_, InitMsgView::protocolVersion(), and protocolVersion_.

Referenced by deserializeAndMergeWithRegistry(), and edm::readHeaderFromStream().

                                                                      {
    if(initView.code() != Header::INIT)
      throw cms::Exception("StreamTranslation","Registry deserialization error")
        << "received wrong message type: expected INIT, got "
        << initView.code() << "\n";

    //Get the process name and store if for Protocol version 4 and above.
    if (initView.protocolVersion() > 3) {

         processName_ = initView.processName();
         protocolVersion_ = initView.protocolVersion();

         FDEBUG(10) << "StreamerInputSource::deserializeRegistry processName = "<< processName_<< std::endl;
         FDEBUG(10) << "StreamerInputSource::deserializeRegistry protocolVersion_= "<< protocolVersion_<< std::endl;
    }

   // calculate the adler32 checksum
   uint32_t adler32_chksum = cms::Adler32((char*)initView.descData(),initView.descLength());
   //std::cout << "Adler32 checksum of init message = " << adler32_chksum << std::endl;
   //std::cout << "Adler32 checksum of init messsage from header = " << initView.adler32_chksum() << " "
   //          << "host name = " << initView.hostName() << " len = " << initView.hostName_len() << std::endl;
    if((uint32)adler32_chksum != initView.adler32_chksum()) {
      std::cerr << "Error from StreamerInputSource: checksum of Init registry blob failed "
                << " chksum from registry data = " << adler32_chksum << " from header = "
                << initView.adler32_chksum() << " host name = " << initView.hostName() << std::endl;
      // skip event (based on option?) or throw exception?
    }

    TClass* desc = getTClass(typeid(SendJobHeader));

    TBufferFile xbuf(TBuffer::kRead, initView.descLength(),
                 (char*)initView.descData(),kFALSE);
    RootDebug tracer(10,10);
    std::auto_ptr<SendJobHeader> sd((SendJobHeader*)xbuf.ReadObjectAny(desc));

    if(sd.get()==0) {
        throw cms::Exception("StreamTranslation","Registry deserialization error")
          << "Could not read the initial product registry list\n";
    }

    return sd;
  }

void edm::StreamerInputSource::fillDescription ( ParameterSetDescription & description ) [static]

Reimplemented from edm::InputSource.

Definition at line 467 of file StreamerInputSource.cc.

Referenced by edm::StreamerFileReader::fillDescriptions().

                                                                    {
    // The default value for "inputFileTransitionsEachEvent" gets defined in the derived class
    // as it depends on the derived class. So, we cannot redefine it here.
    InputSource::fillDescription(desc);
  }

InputSource::ItemType edm::StreamerInputSource::getNextItemType ( ) [private, virtual]

Implements edm::InputSource.

Definition at line 153 of file StreamerInputSource.cc.

References eventCached_, inputFileTransitionsEachEvent_, edm::InputSource::IsEvent, edm::InputSource::IsFile, edm::InputSource::IsLumi, edm::InputSource::IsRun, edm::InputSource::IsStop, edm::InputSource::luminosityBlockAuxiliary(), newLumi_, newRun_, read(), edm::InputSource::resetLuminosityBlockAuxiliary(), edm::InputSource::resetRunAuxiliary(), edm::InputSource::runAuxiliary(), and runEndingFlag_.

                                       {
    if (runEndingFlag_) {
      return IsStop;
    }
    if(newRun_ && runAuxiliary()) {
      return IsRun;
    }
    if(newLumi_ && luminosityBlockAuxiliary()) {
      return IsLumi;
    }
    if (eventCached_) {
      return IsEvent;
    }
    if (inputFileTransitionsEachEvent_) {
      resetRunAuxiliary();
      resetLuminosityBlockAuxiliary();
    }
    read();
    if (!eventCached_) {
      return IsStop;
    } else {
      runEndingFlag_ = false;
      if (inputFileTransitionsEachEvent_) {
        return IsFile;
      }
    }
    if(newRun_) {
      return IsRun;
    } else if(newLumi_) {
      return IsLumi;
    }
    return IsEvent;
  }

void edm::StreamerInputSource::mergeIntoRegistry	(	SendJobHeader const &	header,
		ProductRegistry &	reg,
		bool	subsequent
	)		`[static]`

Definition at line 76 of file StreamerInputSource.cc.

References edm::SendJobHeader::branchIDLists(), buildClassCache(), declareStreamers(), edm::SendJobHeader::descs(), Exception, FDEBUG, edm::fillProductRegistryTransients(), edm::loadExtraClasses(), edm::ProductRegistry::merge(), edm::BranchDescription::Permissive, edm::SendJobHeader::processConfigurations(), edm::BranchIDListHelper::updateFromInput(), and edm::ProductRegistry::updateFromInput().

Referenced by deserializeAndMergeWithRegistry(), and edm::getRegFromFile().

                                                                                                           {

    SendDescs const& descs = header.descs();

    FDEBUG(6) << "mergeIntoRegistry: Product List: " << std::endl;

    if (subsequent) {
      ProductRegistry pReg;
      pReg.updateFromInput(descs);
      fillProductRegistryTransients(header.processConfigurations(), pReg);
      std::string mergeInfo = reg.merge(pReg, std::string(), BranchDescription::Permissive);
      if (!mergeInfo.empty()) {
        throw cms::Exception("MismatchedInput","RootInputFileSequence::previousEvent()") << mergeInfo;
      }
      BranchIDListHelper::updateFromInput(header.branchIDLists(), std::string());
    } else {
      declareStreamers(descs);
      buildClassCache(descs);
      loadExtraClasses();
      reg.updateFromInput(descs);
      fillProductRegistryTransients(header.processConfigurations(), reg);
      BranchIDListHelper::updateFromInput(header.branchIDLists(), std::string());
    }
  }

virtual EventPrincipal* edm::StreamerInputSource::read ( ) [private, pure virtual]

Implemented in edm::EventStreamHttpReader, edm::StreamerInputModule< Producer >, and edm::StreamerFileReader.

Referenced by getNextItemType().

EventPrincipal * edm::StreamerInputSource::readEvent_ ( ) [private, virtual]

Implements edm::InputSource.

Definition at line 144 of file StreamerInputSource.cc.

References eventCached_, edm::InputSource::eventPrincipalCache(), newLumi_, and newRun_.

                                  {
    assert(!newRun_);
    assert(!newLumi_);
    assert(eventCached_);
    eventCached_ = false;
    return eventPrincipalCache();
  }

boost::shared_ptr< FileBlock > edm::StreamerInputSource::readFile_ ( ) [private, virtual]

Reimplemented from edm::InputSource.

Definition at line 71 of file StreamerInputSource.cc.

                                 {
    return boost::shared_ptr<FileBlock>(new FileBlock);
  }

boost::shared_ptr< LuminosityBlockAuxiliary > edm::StreamerInputSource::readLuminosityBlockAuxiliary_ ( ) [private, virtual]

Implements edm::InputSource.

Definition at line 135 of file StreamerInputSource.cc.

References edm::InputSource::luminosityBlockAuxiliary(), newLumi_, and newRun_.

                                                     {
    assert(!newRun_);
    assert(newLumi_);
    assert(luminosityBlockAuxiliary());
    newLumi_ = false;
    return luminosityBlockAuxiliary();
  }

boost::shared_ptr< RunAuxiliary > edm::StreamerInputSource::readRunAuxiliary_ ( ) [private, virtual]

Implements edm::InputSource.

Definition at line 127 of file StreamerInputSource.cc.

References newRun_, and edm::InputSource::runAuxiliary().

                                         {
    assert(newRun_);
    assert(runAuxiliary());
    newRun_ = false;
    return runAuxiliary();
  }

void edm::StreamerInputSource::resetAfterEndRun ( ) [protected]

Definition at line 432 of file StreamerInputSource.cc.

References eventCached_, newLumi_, newRun_, edm::InputSource::reset(), edm::InputSource::resetLuminosityBlockAuxiliary(), edm::InputSource::resetRunAuxiliary(), and runEndingFlag_.

                                             {
     // called from an online streamer source to reset after a stop command
     // so an enable command will work
     resetLuminosityBlockAuxiliary();
     resetRunAuxiliary();
     newRun_ = newLumi_ = true;
     assert(!eventCached_);
     reset();
     runEndingFlag_ = false;
  }

void edm::StreamerInputSource::setEndRun ( ) [inline, protected]

Definition at line 61 of file StreamerInputSource.h.

References runEndingFlag_.

Referenced by edm::EventStreamHttpReader::read().

{runEndingFlag_ = true;}

void edm::StreamerInputSource::setRun ( RunNumber_t r ) [private, virtual]

Reimplemented from edm::InputSource.

Definition at line 443 of file StreamerInputSource.cc.

References Exception, and edm::errors::LogicError.

                                              {
     // Need to define a dummy setRun here or else the InputSource::setRun is called
     // if we have a source inheriting from this and wants to define a setRun method
     throw Exception(errors::LogicError)
     << "StreamerInputSource::setRun()\n"
     << "Run number cannot be modified for this type of Input Source\n"
     << "Contact a Storage Manager Developer\n";
  }

unsigned int edm::StreamerInputSource::uncompressBuffer	(	unsigned char *	inputBuffer,
		unsigned int	inputSize,
		std::vector< unsigned char > &	outputBuffer,
		unsigned int	expectedFullSize
	)		`[static]`

Uncompresses the data in the specified input buffer into the specified output buffer. The inputSize should be set to the size of the compressed data in the inputBuffer. The expectedFullSize should be set to the original size of the data (before compression). Returns the actual size of the uncompressed data. Errors are reported by throwing exceptions.

Definition at line 397 of file StreamerInputSource.cc.

References benchmark_cfg::cerr, Exception, FDEBUG, and runTheMatrix::ret.

Referenced by edm::StreamDQMDeserializer::deserializeDQMEvent(), and deserializeEvent().

                                                                       {
    unsigned long origSize = expectedFullSize;
    unsigned long uncompressedSize = expectedFullSize*1.1;
    FDEBUG(1) << "Uncompress: original size = " << origSize
              << ", compressed size = " << inputSize
              << std::endl;
    outputBuffer.resize(uncompressedSize);
    int ret = uncompress(&outputBuffer[0], &uncompressedSize,
                         inputBuffer, inputSize); // do not need compression level
    //std::cout << "unCompress Return value: " << ret << " Okay = " << Z_OK << std::endl;
    if(ret == Z_OK) {
        // check the length against original uncompressed length
        FDEBUG(10) << " original size = " << origSize << " final size = "
                   << uncompressedSize << std::endl;
        if(origSize != uncompressedSize) {
            std::cerr << "deserializeEvent: Problem with uncompress, original size = "
                 << origSize << " uncompress size = " << uncompressedSize << std::endl;
            // we throw an error and return without event! null pointer
            throw cms::Exception("StreamDeserialization","Uncompression error")
              << "mismatch event lengths should be" << origSize << " got "
              << uncompressedSize << "\n";
        }
    } else {
        // we throw an error and return without event! null pointer
        std::cerr << "deserializeEvent: Problem with uncompress, return value = "
             << ret << std::endl;
        throw cms::Exception("StreamDeserialization","Uncompression error")
            << "Error code = " << ret << "\n ";
    }
    return (unsigned int) uncompressedSize;
  }