LaserSorter Class Reference

#include <LaserSorter.h>

Inheritance diagram for LaserSorter:

Classes
struct	IndexRecord
class	OutStreamRecord
struct	stats_t

Public Member Functions
virtual void	analyze (const edm::Event &, const edm::EventSetup &)
virtual void	beginJob ()
virtual void	beginRun (edm::Run const &, edm::EventSetup const &)
virtual void	endJob ()
	LaserSorter (const edm::ParameterSet &)
	~LaserSorter ()
Public Member Functions inherited from edm::EDAnalyzer
void	callWhenNewProductsRegistered (std::function< void(BranchDescription const &)> const &func)
	EDAnalyzer ()
ModuleDescription const &	moduleDescription () const
std::string	workerType () const
virtual	~EDAnalyzer ()
Public Member Functions inherited from edm::EDConsumerBase
std::vector< ConsumesInfo >	consumesInfo () const
	EDConsumerBase ()
ProductHolderIndexAndSkipBit	indexFrom (EDGetToken, BranchType, TypeID const &) const
void	itemsMayGet (BranchType, std::vector< ProductHolderIndexAndSkipBit > &) const
void	itemsToGet (BranchType, std::vector< ProductHolderIndexAndSkipBit > &) const
std::vector < ProductHolderIndexAndSkipBit > const &	itemsToGetFromEvent () const
void	labelsForToken (EDGetToken iToken, Labels &oLabels) const
void	modulesDependentUpon (const std::string &iProcessName, std::vector< const char * > &oModuleLabels) const
void	modulesWhoseProductsAreConsumed (std::vector< ModuleDescription const * > &modules, ProductRegistry const &preg, std::map< std::string, ModuleDescription const * > const &labelsToDesc, std::string const &processName) const
bool	registeredToConsume (ProductHolderIndex, bool, BranchType) const
bool	registeredToConsumeMany (TypeID const &, BranchType) const
void	updateLookup (BranchType iBranchType, ProductHolderIndexHelper const &)
virtual	~EDConsumerBase ()

Private Types
typedef boost::ptr_list < OutStreamRecord >	OutStreamList

Private Member Functions
void	closeAllStreams ()
void	closeOldStreams (edm::LuminosityBlockNumber_t lumiBlock)
OutStreamList::iterator	closeOutStream (OutStreamList::iterator streamRecord)
OutStreamList::iterator	createOutStream (int fedId, edm::LuminosityBlockNumber_t lumiBlock)
int	dcc2Lme (int dccNum, int dccSide)
int	getDetailedTriggerType (const edm::Handle< FEDRawDataCollection > &rawdata, double *proba=0)
std::vector< int >	getFullyReadoutDccs (const FEDRawDataCollection &data) const
int	getOrbitFromDcc (const edm::Handle< FEDRawDataCollection > &rawdata) const
void	getOutputFedList (const edm::Event &event, const FEDRawDataCollection &data, std::vector< unsigned > &fedIds) const
OutStreamRecord *	getStream (int fedId, edm::LuminosityBlockNumber_t lumiBlock)
bool	isDccEventEmpty (const FEDRawData &data, size_t *dccLen=0, int *nTowerBlocks=0) const
int	readFormatVersion (std::ifstream &in, const std::string &fileName)
bool	readIndexTable (std::ifstream &in, std::string &inName, OutStreamRecord &outRcd, std::string *err)
bool	renameAsBackup (const std::string &fileName, std::string &newFileName)
void	restoreStreamsOfLumiBlock (int lumiBlock)
void	streamFileName (int fedId, edm::LuminosityBlockNumber_t lumiBlock, std::string &tmpName, std::string &finalName)
bool	writeEvent (OutStreamRecord &out, const edm::Event &event, int detailedTriggerType, const FEDRawDataCollection &data)
bool	writeEventHeader (std::ofstream &out, const edm::Event &evt, int fedId, unsigned nFeds)
bool	writeFedBlock (std::ofstream &out, const FEDRawData &data)
void	writeFileHeader (std::ofstream &out)
bool	writeIndexTable (std::ofstream &out, std::vector< IndexRecord > &indices)

Static Private Member Functions
static std::string	toString (uint64_t t)

Private Attributes
int	detailedTrigType_
bool	disableOutput_
bool	doOutputList_
edm::InputTag	fedRawDataCollectionTag_
edm::EDGetTokenT < FEDRawDataCollection >	fedRawDataCollectionToken_
std::vector< std::string >	fedSubDirs_
unsigned char	formatVersion_
int	iNoEcalDataMess_
int	iNoFullReadoutDccError_
std::ofstream	logFile_
edm::LuminosityBlockNumber_t	lumiBlock_
edm::LuminosityBlockNumber_t	lumiBlockPrev_
int	lumiBlockSpan_
int	maxFullReadoutDccError_
int	maxNoEcalDataMess_
int	orbit_
int	orbitCountInALumiBlock_
struct timeval	orbitZeroTime_
std::string	outputDir_
std::ofstream	outputList_
std::string	outputListFile_
OutStreamList	outStreamList_
bool	overWriteLumiBlockId_
edm::RunNumber_t	runNumber_
struct LaserSorter::stats_t	stats_
std::ofstream	timeLog_
std::string	timeLogFile_
timeval	timer_
bool	timing_
int	verbosity_

Static Private Attributes
static const int	ecalDccFedIdMax_ = 654
static const int	ecalDccFedIdMin_ = 601
static const int	indexOffset32_ = 1
static const int	matacqFedId_ = 655
static const unsigned	maxEvents_ = 1<<20
static const stats_t	stats_init = {0, 0, 0, 0, 0}

Additional Inherited Members
Public Types inherited from edm::EDAnalyzer
typedef EDAnalyzer	ModuleType
Static Public Member Functions inherited from edm::EDAnalyzer
static const std::string &	baseType ()
static void	fillDescriptions (ConfigurationDescriptions &descriptions)
static void	prevalidate (ConfigurationDescriptions &)
Protected Member Functions inherited from edm::EDConsumerBase
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	consumes (edm::InputTag const &tag)
EDGetToken	consumes (const TypeToGet &id, edm::InputTag const &tag)
template<BranchType B>
EDGetToken	consumes (TypeToGet const &id, edm::InputTag const &tag)
ConsumesCollector	consumesCollector ()
	Use a ConsumesCollector to gather consumes information from helper functions. More...
template<typename ProductType , BranchType B = InEvent>
void	consumesMany ()
void	consumesMany (const TypeToGet &id)
template<BranchType B>
void	consumesMany (const TypeToGet &id)
template<typename ProductType , BranchType B = InEvent>
EDGetTokenT< ProductType >	mayConsume (edm::InputTag const &tag)
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)
template<BranchType B>
EDGetToken	mayConsume (const TypeToGet &id, edm::InputTag const &tag)

Detailed Description

This module is used to classify events of laser sequence acquired in a global run. Sorting: Events are grouped by bunch of consecutive events from the same FED. A file of such a bunch of events is identified by the FED or ECAL sector and the luminosity block id of the first event.

Sorting strategy: It is assumes that:

• one sequence FED does not overlap more than two lumi blocks.
• lumi blocks are read in order A FED file will be closed as soon as one event 2 lumi block ahead from the first event feed in the file is read.

File completion: while being feeding, .part is appended at the end of the each output file.Once a file is completed (see above), it is renamed without the enclosing .part suffix.

Definition at line 44 of file LaserSorter.h.

Member Typedef Documentation

typedef boost::ptr_list<OutStreamRecord> LaserSorter::OutStreamList

private

Definition at line 117 of file LaserSorter.h.

Constructor & Destructor Documentation

LaserSorter::LaserSorter

(

const edm::ParameterSet &

pset

)

Definition at line 80 of file LaserSorter.cc.

References gather_cfg::cout, doOutputList_, edm::hlt::Exception, fedRawDataCollectionTag_, fedRawDataCollectionToken_, fedSubDirs_, logFile_, now(), dbtoconf::out, outputDir_, outputList_, outputListFile_, timeLog_, timeLogFile_, timer_, and timing_.

  : lumiBlock_(0),
    lumiBlockPrev_(0),
    formatVersion_(5),
    outputDir_(pset.getParameter<std::string>("outputDir")),
    fedSubDirs_(pset.getParameter<std::vector<std::string> >("fedSubDirs")),
    timeLogFile_(pset.getUntrackedParameter<std::string>("timeLogFile", "")),
    disableOutput_(pset.getUntrackedParameter<bool>("disableOutput", false)),
    runNumber_(0),
    outputListFile_(pset.getUntrackedParameter<string>("outputListFile", "")),
    doOutputList_(false),
    verbosity_(pset.getUntrackedParameter<int>("verbosity", 0)),
    iNoFullReadoutDccError_(0),
    maxFullReadoutDccError_(pset.getParameter<int>("maxFullReadoutDccError")),
    iNoEcalDataMess_(0),
    maxNoEcalDataMess_(pset.getParameter<int>("maxNoEcalDataMess")),
    lumiBlockSpan_(pset.getParameter<int>("lumiBlockSpan")),
    fedRawDataCollectionTag_(pset.getParameter<edm::InputTag>("fedRawDataCollectionTag")),
    stats_(stats_init),
    overWriteLumiBlockId_(pset.getParameter<bool>("overWriteLumiBlockId")),
    orbitCountInALumiBlock_(pset.getParameter<int>("orbitCountInALumiBlock")),
    orbit_(-1),
    orbitZeroTime_(nullTime)
{

  gettimeofday(&timer_, 0);
  logFile_.open("eventSelect.log", ios::app | ios::out); 
  
  const unsigned nEcalFeds= 54;
  if(fedSubDirs_.size()!= nEcalFeds+1){
    throw cms::Exception("LaserSorter") 
      << "Configuration error: "
      << "fedSubDirs parameter must be a vector "
      << " of " << nEcalFeds << " strings"
      << " (subdirectory for unknown triggered FED followed by "
      "subdirectories for FED ID 601 "
      "to FED ID 654 in increasing FED ID order)";
  }

  fedRawDataCollectionToken_ = consumes<FEDRawDataCollection>(fedRawDataCollectionTag_);
  
  if(outputListFile_.size()!=0){
    outputList_.open(outputListFile_.c_str(), ios::app);
    if(outputList_.bad()){
      throw cms::Exception("FileOpen")
    << "Failed to open file '" << outputListFile_
    << "' for logging of output file path list.";
    }
    doOutputList_ = true;
  }
  
  if(timeLogFile_.size()>0){
    timeLog_.open(timeLogFile_.c_str());
    if(timeLog_.fail()){
      cout << "[LaserSorter " << now() << "] "
           << "Failed to open file " << timeLogFile_ << " to log timing.\n";
      timing_ = false;
    } else{
      timing_ = true;
    }
  }

  struct stat fileStat;
  if(0==stat(outputDir_.c_str(), &fileStat)){
    if(!S_ISDIR(fileStat.st_mode)){
      throw cms::Exception("[LaserSorter]")
        << "File " << outputDir_ << " exists but is not a directory "
        << " as expected.";
    }
  } else {//directory does not exists, let's try to create it
    if(0!=mkdir(outputDir_.c_str(), 0755)){
      throw cms::Exception("[LaserSorter]")
        << "Failed to create directory " << outputDir_
        << " for writing data.";
    }
  }

  logFile_ << "# "
    "run\t"
    "LB\t" 
    "event\t" 
    "trigType\t"
    "FED\t"
    "side\t"
    "LB out\t"
    "Written\t"
    "ECAL data\n";
}

LaserSorter::~LaserSorter

(

)

Definition at line 169 of file LaserSorter.cc.

References logFile_, LaserSorter::stats_t::nInvalidDccStrict, LaserSorter::stats_t::nInvalidDccWeak, LaserSorter::stats_t::nRead, LaserSorter::stats_t::nRestoredDcc, LaserSorter::stats_t::nWritten, and stats_.

                         {
  logFile_ << "Summary. Event count: "
           << stats_.nRead << " processed, "
           << stats_.nWritten << " written, "
           << stats_.nInvalidDccStrict << " with errors in DCC ID values, "
           << stats_.nInvalidDccWeak << " with unusable DCC ID values, "
           << stats_.nRestoredDcc << " restored DCC ID based on DCC block size\n";
}

Member Function Documentation

void LaserSorter::analyze ( const edm::Event &  event, const edm::EventSetup &  es  )

virtual

Implements edm::EDAnalyzer.

Definition at line 181 of file LaserSorter.cc.

References closeAllStreams(), closeOldStreams(), create_public_lumi_plots::color, colorNames, gather_cfg::cout, ecaldqm::dccId(), detailedTrigNames, detailedTrigType_, disableOutput_, ecalDccFedIdMax_, ecalDccFedIdMin_, LaserSorter::OutStreamRecord::excludedOrbit(), fedRawDataCollectionToken_, LaserSorter::OutStreamRecord::finalFileName(), getDetailedTriggerType(), getFullyReadoutDccs(), getOrbitFromDcc(), getStream(), i, edm::EventBase::id(), iNoEcalDataMess_, iNoFullReadoutDccError_, logFile_, lumiBlock_, lumiBlockPrev_, lumiBlockSpan_, maxFullReadoutDccError_, csvLumiCalc::maxLumi, csvLumiCalc::minLumi, LaserSorter::stats_t::nInvalidDccStrict, LaserSorter::stats_t::nInvalidDccWeak, now(), LaserSorter::stats_t::nRead, LaserSorter::stats_t::nRestoredDcc, LaserSorter::stats_t::nWritten, orbit_, orbitCountInALumiBlock_, dbtoconf::out, overWriteLumiBlockId_, lumiPlot::rawdata, restoreStreamsOfLumiBlock(), edm::EventID::run(), runNumber_, LaserSorter::OutStreamRecord::startingLumiBlock(), stats_, edmStreamStallGrapher::t, timeLog_, timer_, timing_, verbosity_, and writeEvent().

                                                                  {
  if(timing_){
    timeval t;
    gettimeofday(&t, 0);
    timeLog_ << t.tv_sec << "."
             << setfill('0') << setw(3) << (t.tv_usec+500)/1000 << setfill(' ')
             << "\t"
             << (t.tv_usec - timer_.tv_usec)*1. 
      + (t.tv_sec - timer_.tv_sec)*1.e6 << "\t";
    timer_ = t;
  } 

  ++stats_.nRead;
  
  if(event.id().run()!=runNumber_){//run changed or first event
    //for a new run, starts with a new output stream set.
    closeAllStreams();
    runNumber_ = event.id().run();
    iNoFullReadoutDccError_ = 0;
    iNoEcalDataMess_ = 0;
    lumiBlockPrev_ = 0;
    lumiBlock_ = 0;
  }
  
  edm::Handle<FEDRawDataCollection> rawdata;
  event.getByToken(fedRawDataCollectionToken_, rawdata);  

  //orbit number
  //FIXME: orbit from edm Event is currently wrong. Forcing to use of CMS orbit until
  //it is fixed. See https://hypernews.cern.ch/HyperNews/CMS/get/commissioning/5343/2.html
#if 0
  orbit_ = event.orbitNumber();
#else
  orbit_ = -1;
#endif

  //  std::cerr << "Orbit ID CMS, ECAL: " << orbit_ << "\t" << getOrbitFromDcc(rawdata) << "\n";

  if(orbit_ < 0){ //For local run CMSSW failed to find the orbit number
    //    cout << "Look for orbit from DCC headers....\n";
    orbit_ = getOrbitFromDcc(rawdata);
  }

  
  //The "detailed trigger type DCC field content:
  double dttProba = 0;
  int dtt = getDetailedTriggerType(rawdata, &dttProba);

  if(overWriteLumiBlockId_){
    edm::LuminosityBlockNumber_t lb = lumiBlock_;
    lumiBlock_ = orbit_ / orbitCountInALumiBlock_;
    if(lb!=lumiBlock_){
      std::cout << "[LaserSorter " << now() << "] Overwrite LB mode. LB number changed from: " << lb << " to " <<lumiBlock_<< "\n";
    }
  } else{
    edm::LuminosityBlockNumber_t lb = lumiBlock_;
    lumiBlock_ = event.luminosityBlock();
    if(lb!=lumiBlock_){
      std::cout << "[LaserSorter " << now() << "] Standard LB mode. LB number changed from: " << lb << " to " <<lumiBlock_<< "\n";
     }
  }

  detailedTrigType_ = dtt;
  const int trigType =   (detailedTrigType_ >>8 ) & 0x7;
  const int color    =   (detailedTrigType_ >>6 ) & 0x3;
  const int dccId    =   (detailedTrigType_ >>0 ) & 0x3F;
  int triggeredFedId =   (detailedTrigType_ == -2) ? -1 : (600 + dccId);
  const int side     =   (detailedTrigType_ >>11) & 0x1;
  //monitoring region extended id:
  //  const int lme = dcc2Lme(dccId, side);

  if(detailedTrigType_ > -2){
    if(dttProba < 1. || triggeredFedId < ecalDccFedIdMin_
       || triggeredFedId > ecalDccFedIdMax_){
      ++stats_.nInvalidDccStrict;
    }

    if(triggeredFedId<ecalDccFedIdMin_ || triggeredFedId > ecalDccFedIdMax_){
      if(verbosity_) cout <<  "[LaserSorter " << now() << "] " << "DCC ID (" << dccId
                          << ") found in trigger type is out of range.";
      ++stats_.nInvalidDccWeak;
      vector<int> ids = getFullyReadoutDccs(*rawdata);
      if(ids.size()==0){
        if(verbosity_ && iNoFullReadoutDccError_ < maxFullReadoutDccError_){
          cout << " No fully read-out DCC found\n";
          ++iNoFullReadoutDccError_;
        }
      } else if(ids.size()==1){
        triggeredFedId = ids[0];
        if(verbosity_) cout << " ID guessed from DCC payloads\n";
        ++stats_.nRestoredDcc;
      } else{ //ids.size()>1
        if(verbosity_){
          cout << " Several fully read-out Dccs:";
          for(unsigned i=0; i < ids.size(); ++i) cout << " " << ids[i];
          cout << "\n";
        }
      }
    }
  
    if(verbosity_>1) cout << "\n----------------------------------------------------------------------\n"
                          << "Event id: " 
                          << " " << event.id() << "\n"
                          << "Lumin block: " << lumiBlock_ << "\n"
                          << "TrigType: " << detailedTrigNames[trigType&0x7]
                          << " Color: " << colorNames[color&0x3]
                          <<  " FED: " << triggeredFedId
                          << " side:" << side << "\n"
                          << "\n----------------------------------------------------------------------\n";
    
  } else{ //NO ECAL DATA
    if(verbosity_>1) cout << "\n----------------------------------------------------------------------\n"
                          << "Event id: " 
                          << " " << event.id() << "\n"
                          << "Lumin block: " << lumiBlock_ << "\n"
                          << "No ECAL data\n"
                          << "\n----------------------------------------------------------------------\n";
  }
  
  logFile_ << event.id().run() << "\t"
           << lumiBlock_ << "\t" 
           << event.id().event() << "\t" 
           << trigType << "\t"
           << triggeredFedId << "\t"
           << side;
    
  bool written = false;
  int assignedLB = -1;
  
  if(lumiBlock_!=lumiBlockPrev_){
    //lumi block change => need for stream garbage collection
    const int lb = lumiBlock_;
    closeOldStreams(lb);
    int minLumi = lumiBlock_ - lumiBlockSpan_;
    int maxLumi = lumiBlock_ + lumiBlockSpan_;
    for(int lb1 = minLumi; lb1 <= maxLumi; ++lb1){
      restoreStreamsOfLumiBlock(lb1);
    }
  }
    
//     if(lumiBlock_ < lumiBlockPrev_){
//       throw cms::Exception("LaserSorter") 
//         << "Process event has a lumi block (" << lumiBlock_ << ")"
//         << "older than previous one (" << lumiBlockPrev_ << "). "
//         << "This can be due by wrong input file ordering or bad luminosity "
//         << "block indication is the event header. "
//         << "Event cannot be processed";
//     }

    if(disableOutput_){
      /* NO OP*/
    } else{
      OutStreamRecord* out = getStream(triggeredFedId, lumiBlock_);

      if(out!=0){
        assignedLB = out->startingLumiBlock();
        if(out->excludedOrbit().find(orbit_)
           ==out->excludedOrbit().end()){
          if(verbosity_ > 1) cout << "[LaserSorter " << now() << "] "
                              << "Writing out event from FED " << triggeredFedId 
                              << " LB " << lumiBlock_
                              << " orbit " << orbit_ << "\n";
          int dtt = (detailedTrigType_ >=0) ? detailedTrigType_ : -1; //shall we use -1 or 0 for undefined value?
          written = written
            || writeEvent(*out, event, dtt, *rawdata);
          ++stats_.nWritten;
        } else{
          if(verbosity_) cout << "[LaserSorter " << now() << "] "
                              << "File " << out->finalFileName() << " "
                              << "already contains calibration event from FED "
                              << triggeredFedId << ", LB = "
                              << lumiBlock_
                              << " with orbit ID "
                              << orbit_ << ". Event skipped.\n";
        }
      }
    }
    lumiBlockPrev_ = lumiBlock_;
    
    logFile_ << "\t";
    if(assignedLB>=0) logFile_ << assignedLB; else logFile_ << "-";
    logFile_ << "\t" << (written?"Y":"N") << "\n"; 
    logFile_  << "\t" << (detailedTrigType_==-2?"N":"Y") << "\n";
    
    if(timing_){
      timeval t;
      gettimeofday(&t, 0);
      timeLog_ << (t.tv_usec - timer_.tv_usec)*1. 
        + (t.tv_sec - timer_.tv_sec)*1.e6 << "\n";
      timer_ = t;
    }
}

void LaserSorter::beginJob ( void  )

virtual

Reimplemented from edm::EDAnalyzer.

Definition at line 951 of file LaserSorter.cc.

                          {
}

void LaserSorter::beginRun ( edm::Run const &  run, edm::EventSetup const &  es  )

virtual

Reimplemented from edm::EDAnalyzer.

Definition at line 1217 of file LaserSorter.cc.

                                                                    {
  //  cout << "Run starts at :" << run.runAuxiliary().beginTime().value() << "\n";
}

void LaserSorter::closeAllStreams ( )

private

Closes all opened output streams.

Definition at line 468 of file LaserSorter.cc.

References closeOutStream(), and outStreamList_.

Referenced by analyze(), and endJob().

                                 {
  for(OutStreamList::iterator it = outStreamList_.begin();
      it != outStreamList_.end();/*NOOP*/){
    it = closeOutStream(it);
  }
}

void LaserSorter::closeOldStreams ( edm::LuminosityBlockNumber_t  lumiBlock )

private

Closes output stream 2 lumi block older than the input 'lumiBlock' ID.

Parameters

lumiBlock

ID of the reference luminosity block.

Definition at line 475 of file LaserSorter.cc.

References closeOutStream(), gather_cfg::cout, lumiBlockSpan_, now(), outStreamList_, and verbosity_.

Referenced by analyze().

                                                                     {
  const edm::LuminosityBlockNumber_t minLumiBlock = lumiBlock - lumiBlockSpan_;
  const edm::LuminosityBlockNumber_t maxLumiBlock = lumiBlock + lumiBlockSpan_;
  //If container type is ever changed, beware that
  //closeOutStream call in the loop removes it from outStreamList
  for(boost::ptr_list<OutStreamRecord>::iterator it = outStreamList_.begin();
      it != outStreamList_.end();
      /*NOOP*/){
    if(it->startingLumiBlock() < minLumiBlock
       || it->startingLumiBlock() > maxLumiBlock){
      //event older than 2 lumi block => stream can be closed
      if(verbosity_) cout << "[LaserSorter " << now() << "] "
                       << "Closing file for "
                       << "FED " 
                       << it->fedId()
                       << " LB " << it->startingLumiBlock()
                       << "\n";
      it = closeOutStream(it);
    } else{
      ++it;
    }
  }
}

LaserSorter::OutStreamList::iterator LaserSorter::closeOutStream ( OutStreamList::iterator  streamRecord )

private

Closes an output stream and removed it from opened stream records. Beware: this methode modifies outStreamList_.

Parameters

streamRecord

record of the output stream to close.

Returns

iterator to element next to the deleted one.

Definition at line 897 of file LaserSorter.cc.

References gather_cfg::cout, doOutputList_, f, analyzePatCleaning_cfg::inputFile, now(), outputList_, outStreamList_, AlCaHLTBitMon_QueryRunRegistry::string, edmStreamStallGrapher::t, cond::rpcobgas::time, verbosity_, and writeIndexTable().

Referenced by closeAllStreams(), and closeOldStreams().

                                         {
  if(streamRecord==outStreamList_.end()) return outStreamList_.end();
  
  if(verbosity_) cout << "[LaserSorter " << now() << "] " << "Writing Index table of file "
                      << streamRecord->finalFileName() << "\n";
  ofstream& out = *streamRecord->out();
  out.clear();
  if(!writeIndexTable(out, *streamRecord->indices())){
    cout << "Error while writing index table for file "
         << streamRecord->finalFileName() << ". "
         << "Resulting file might be corrupted. "
         << "The error can be due to a lack of disk space.";
  }
  
  if(verbosity_) cout << "[LaserSorter " << now() << "] " << "Closing file " 
                      << streamRecord->finalFileName() << ".\n";
  out.close();
  
  const std::string& tmpFileName = streamRecord->tmpFileName();
  const std::string& finalFileName = streamRecord->finalFileName();

  if(verbosity_) cout << "[LaserSorter " << now() << "] " << "Renaming " << tmpFileName
                      << " to "    << finalFileName << ".\n";

  if(0!=rename(tmpFileName.c_str(), finalFileName.c_str())){
    cout << "[LaserSorter " << now() << "] "
         << " Failed to rename output file from "
         << tmpFileName << " to "    << finalFileName
         << ". " << strerror(errno) << "\n";
  }

  if(doOutputList_){
    char buf[256];
    time_t t = time(0);
    strftime(buf, sizeof(buf), "%F %R:%S", localtime(&t));

    ifstream f(".watcherfile");
    string inputFile;
    f >> inputFile;
    outputList_ << finalFileName << "\t" << buf
                << "\t" << inputFile
                << endl;
  }
  
  return outStreamList_.erase(streamRecord);
}

LaserSorter::OutStreamList::iterator LaserSorter::createOutStream ( int  fedId, edm::LuminosityBlockNumber_t  lumiBlock  )

private

Creates an output stream. It must be ensured before calling this method that the output stream is not already opened (See outStreamList_).

Parameters

fedId	FED ID of the event to stream out.
lumiBlock	starting lumi block of the event group to write to the stream.

Returns

iterator to the new stream record. outStreamList_.end() in case of failure.

Definition at line 640 of file LaserSorter.cc.

References EnergyCorrector::c, gather_cfg::cout, TauDecayModes::dec, end, edm::hlt::Exception, formatVersion_, recoMuon::in, min(), now(), dbtoconf::out, outStreamList_, readFormatVersion(), readIndexTable(), renameAsBackup(), streamFileName(), AlCaHLTBitMon_QueryRunRegistry::string, estimatePileup_makeJSON::trunc, verbosity_, and writeFileHeader().

Referenced by getStream().

                                                                  {
  if(verbosity_) cout << "[LaserSorter " << now() << "] " << "Creating a stream for FED " << fedId
                      << " lumi block " << lumiBlock << ".\n";
  std::string tmpName;
  std::string finalName;

  streamFileName(fedId, lumiBlock, tmpName, finalName);

  errno = 0;

  //checks if a file with tmpName name already exists:
  ofstream* out = new ofstream(tmpName.c_str(), ios::out | ios::in);  
  if(out->is_open()){//temporary file already exists. Making a backup:
    string newName;
    if(!renameAsBackup(tmpName, newName)){
      throw cms::Exception("LaserSorter")
        << "Failed to rename file " << tmpName
        << "  to " << newName << "\n";
    }
    if(verbosity_) cout << "[LaserSorter " << now() << "] "
                        << "Already existing File " << tmpName
                        << " renamed to "
                        << newName << "\n";
    out->close();
  }

  out->clear();
  out->open(tmpName.c_str(), ios::out | ios::trunc);
  
  if(out->fail()){//failed to create file
    delete out;
    throw cms::Exception("LaserSorter")
      << "Failed to create file " 
      << tmpName << " for writing event from FED " << fedId
      << " lumi block " << lumiBlock
      << ": "  << strerror(errno) << "\n.";
  }
  
  ifstream in(finalName.c_str());
  bool newFile = true;
  if(in.good()){//file already exists with final name.
    if(verbosity_) cout << "[LaserSorter " << now() << "] " << "File "
                        << finalName
                        << " already exists. It will be updated if needed.\n";
    //Copying its contents:
    char buffer[256];
    streamsize nread = -1;
    int vers = readFormatVersion(in, finalName);
    if(vers==-1){
      edm::LogWarning("LaserSorter") << "File " << tmpName.c_str()
                                     << " is not an LMF file despite its extension or "
                                     << "it is corrupted.\n";
    } else if(vers!=formatVersion_){
      edm::LogWarning("LaserSorter") << "Cannot include events already in file "
                                     << tmpName.c_str()
                                     << " because of version "
                                     << "mismatch (found version "
                                     << (int)vers << " while "
                                     << "only version "
                                     << (int)formatVersion_
                                     << " is supported).\n";
    } else{
      newFile = false;
      //read index table offset value:
      const int indexTableOffsetPos8 = 1*sizeof(uint32_t);
      uint32_t indexTableOffsetValue = 0;
      in.clear();
      in.seekg(indexTableOffsetPos8, ios::beg);
      in.read((char*) &indexTableOffsetValue,
              sizeof(indexTableOffsetValue));
      if(in.fail()){
        cout << "[LaserSorter " << now() << "] " << "Failed to read offset of index table "
          " in the existing file " << finalName << "\n";
      } else{
        if(verbosity_>2) cout << "[LaserSorter " << now() << "] " << "Index table offset of "
                           "original file " << finalName << ": 0x"
                              << hex << setfill('0')
                              << setw(8) << indexTableOffsetValue
                              << dec << setfill(' ') << "\n";
      }
      in.clear();
      in.seekg(0, ios::beg);

      //copy legacy file contents except the index table
      uint32_t toRead = indexTableOffsetValue;
      cout << "[LaserSorter " << now() << "] " << "Copying " << finalName << " to " << tmpName << endl;
      while(!in.eof()
            && (toRead > 0)
            && (nread=in.readsome(buffer, min(toRead, (uint32_t)sizeof(buffer))))!=0){
        //         cout << "Writing " << nread << " bytes to file "
        //              << tmpName.c_str() << "\n";
        toRead -= nread;
        // out->seekp(0, ios::end);
        out->write(buffer, nread);
        if(out->bad()){
          throw cms::Exception("LaserSorter") << "Error while writing to file "
                                              << tmpName
                                              << ". Check if there is enough "
                                              << "space on the device.\n";
        }
      }

      //resets index table offset field:
      indexTableOffsetValue = 0 ;
      out->clear();
      out->seekp(indexTableOffsetPos8, ios::beg);
      out->write((char*)&indexTableOffsetValue, sizeof(uint32_t));
      out->clear();
      out->seekp(0, ios::end);
    }
  }

#if 0
  out->flush();
  cout << "Press enter... file name was " << tmpName << endl;
  char c;
  cin >> c;
#endif
  
  OutStreamRecord* outRcd = new OutStreamRecord(fedId, lumiBlock,
                                                out,
                                                tmpName, finalName);
  
  if(newFile){
    writeFileHeader(*out);
  } else{
    std::string errMsg;
    if(!readIndexTable(in, finalName, *outRcd, &errMsg)){
      throw cms::Exception("LaserSorter") << errMsg << "\n";
    }
  }

  return outStreamList_.insert(outStreamList_.end(), outRcd);
}

int LaserSorter::dcc2Lme ( int  dccNum, int  dccSide  )

private

Definition at line 374 of file LaserSorter.cc.

References min().

                                         {
  int fedid = (dcc%600) + 600; //to handle both FED and DCC id.
  vector<int> lmes;
  // EE -
  if( fedid <= 609 ) {
    if ( fedid <= 607 ) {
      lmes.push_back(fedid-601+83);
    } else if ( fedid == 608 ) {
      lmes.push_back(90);
      lmes.push_back(91);
    } else if ( fedid == 609 ) {
      lmes.push_back(92);
    }
  } //EB
  else if ( fedid >= 610  && fedid <= 645 ) {
    lmes.push_back(2*(fedid-610)+1);
    lmes.push_back(lmes[0]+1);
  } // EE+ 
  else if ( fedid >= 646 ) {
    if ( fedid <= 652 ) {
      lmes.push_back(fedid-646+73);
    } else if ( fedid == 653 ) {
      lmes.push_back(80);
      lmes.push_back(81);
    } else if ( fedid == 654 ) {
      lmes.push_back(82);
    }
  }
  return lmes.size()==0?-1:lmes[min(lmes.size(), (size_t)side)];
}

void LaserSorter::endJob ( void  )

virtual

Reimplemented from edm::EDAnalyzer.

Definition at line 945 of file LaserSorter.cc.

References closeAllStreams().

                        {
  //TODO: better treatement of last files:
  //they might be imcomplete...
  closeAllStreams();
}

int LaserSorter::getDetailedTriggerType ( const edm::Handle< FEDRawDataCollection > &  rawdata, double *  proba = 0  )

private

Retrieve detailed trigger type (trigger type, DCC, side) from raw event

Parameters

rawdata	FED data collection
proba	if not null used to store the maximum of occurence frequency of the detailed trigger types (DTT) appearing in each DCC block. In normal condition every DCC indicated the same DTT and this value is 1.

Returns

detailed trigger type. In case of descripancy between the DCCs, the most frequent value is returned if it covers more than 80% of the present DCC blocks, -1 is returned otherwise. If event does not contain any ECAL data -2 is returned.

Definition at line 427 of file LaserSorter.cc.

References Majority< T >::add(), gather_cfg::cout, FEDRawData::data(), data, ecalDccFedIdMax_, ecalDccFedIdMin_, iNoEcalDataMess_, FEDNumbering::inRange(), maxNoEcalDataMess_, now(), AlCaHLTBitMon_ParallelJobs::p, Majority< T >::result(), FEDRawData::size(), and verbosity_.

Referenced by analyze().

                                                      {  
  Majority<int> stat;
  bool ecalData = false;
  for(int id=ecalDccFedIdMin_; id<=ecalDccFedIdMax_; ++id){
    if(!FEDNumbering::inRange(id)) continue;
    const FEDRawData& data = rawdata->FEDData(id);
    const int detailedTrigger32 = 5;
    if(verbosity_>3) cout << "[LaserSorter " << now() << "] " 
                          << "FED " << id << " data size: "  
                          << data.size() << "\n"; 
    if(data.size()>=4*(detailedTrigger32+1)){
      ecalData = true;
      const uint32_t* pData32 = (const uint32_t*) data.data();
      int tType = pData32[detailedTrigger32] & 0xFFF;
      if(verbosity_>3) cout << "[LaserSorter " << now() << "] "
                            << "Trigger type " << tType << "\n";
      stat.add(tType);
    }
  }
  if(!ecalData) return -2;
  double p;
  int tType = stat.result(&p);
  if(p<0){
    //throw cms::Exception("NotFound") << "No ECAL DCC data found\n";
    if(iNoEcalDataMess_ < maxNoEcalDataMess_){
      edm::LogWarning("NotFound")  << "No ECAL DCC data found. "
        "(This warning will be disabled for the current run after "
                                   << maxNoEcalDataMess_ << " occurences.)";
      ++iNoEcalDataMess_;
    }
    tType = -1;
  } else if(p<.8){
    //throw cms::Exception("EventCorruption") << "Inconsitency in detailed trigger type indicated in ECAL DCC data headers\n";
    edm::LogWarning("EventCorruption") << "Inconsitency in detailed trigger type indicated in ECAL DCC data headers\n";
    tType = -1;
  }
  if(proba) *proba = p;
  return tType;
}

std::vector< int > LaserSorter::getFullyReadoutDccs ( const FEDRawDataCollection &  data ) const

private

Retrieves DCCs which were fully read out (>=68 readout channels).

Parameters

data	DCC data collection

Returns

FED ids.

Definition at line 1025 of file LaserSorter.cc.

References ecalDccFedIdMax_, ecalDccFedIdMin_, FEDRawDataCollection::FEDData(), isDccEventEmpty(), ecaldqm::nTowers, and query::result.

Referenced by analyze().

                                                                      {
  int nTowers;
  vector<int> result;
  for(int fed = ecalDccFedIdMin_; fed <= ecalDccFedIdMax_; ++fed){
    const FEDRawData& fedData = data.FEDData(fed);
    isDccEventEmpty(fedData, 0, &nTowers);
    if(nTowers>=68) result.push_back(fed);
  }
  return result;
}

int LaserSorter::getOrbitFromDcc ( const edm::Handle< FEDRawDataCollection > &  rawdata ) const

private

Retrieves the orbit ID from DCC headers. The first header found is used. the raw data where to read the orbit id.

Returns

the event orbit ID or -1 if it was not found

Definition at line 405 of file LaserSorter.cc.

References FEDRawData::data(), data, ecalDccFedIdMax_, ecalDccFedIdMin_, FEDNumbering::inRange(), and FEDRawData::size().

Referenced by analyze().

                                                                                    {
  const int orbit32 = 6;
  for(int id=ecalDccFedIdMin_; id<=ecalDccFedIdMax_; ++id){
    if(!FEDNumbering::inRange(id)) continue;
    const FEDRawData& data = rawdata->FEDData(id);
    if(data.size() >= 4*(orbit32+1)){
      const uint32_t* pData32 = (const uint32_t*) data.data();
//      cout << "Found a DCC header: "
//           << pData32[0] << " "
//           << pData32[1] << " "
//           << pData32[2] << " "
//           << pData32[3] << " "
//           << pData32[4] << " "
//           << pData32[5] << " "
//           << pData32[6] << " "
//           << "\n";
      return pData32[orbit32];
    }
  }
  return -1;
}

void LaserSorter::getOutputFedList ( const edm::Event &  event, const FEDRawDataCollection &  data, std::vector< unsigned > &  fedIds  ) const

private

Computes the list of FEDs which data must be written out.

Parameters

data	CMS raw event
fedIds	[out] list of FEDs to keep

Definition at line 997 of file LaserSorter.cc.

References ecalDccFedIdMax_, ecalDccFedIdMin_, FEDRawDataCollection::FEDData(), isDccEventEmpty(), matacqFedId_, and FEDRawData::size().

Referenced by writeEvent().

                                                                     {
  fedIds.erase(fedIds.begin(), fedIds.end());
  for(int id = ecalDccFedIdMin_; id <= ecalDccFedIdMax_; ++id){
    size_t dccLen;
    const FEDRawData& dccEvent = data.FEDData(id);
    if(!isDccEventEmpty(dccEvent, &dccLen)){
      fedIds.push_back(id);
    }
    if(dccLen*sizeof(uint64_t)!=dccEvent.size()){
      edm::LogWarning("LaserSorter")
        << "Length error in data of FED " << id
        << " in event " << event.id() 
        << ", Data of this FED dropped.";
    }
  }
  //   cout << __FILE__ << ":" << __LINE__ << ": "
  //        <<  "data.FEDData(" << matacqFedId_ << ").size() = "
  //        <<  data.FEDData(matacqFedId_).size() << "\n";
  if(data.FEDData(matacqFedId_).size()>4){//matacq block present
    //    cout << __FILE__ << ":" << __LINE__ << ": "
    //     <<  "Adding matacq to list of FEDs\n";
    fedIds.push_back(matacqFedId_);
  }
}

LaserSorter::OutStreamRecord * LaserSorter::getStream ( int  fedId, edm::LuminosityBlockNumber_t  lumiBlock  )

private

Gets and eventually creates the output stream for writing the events of a given FED and luminosity block.

Parameters

fedId	ID of the FED the event is issued from
lumiBlock	luminositu block of the event

Returns

pointer of the output stream record or null if not found.

Definition at line 500 of file LaserSorter.cc.

References funct::abs(), gather_cfg::cout, createOutStream(), ecalDccFedIdMax_, lumiBlockSpan_, now(), outStreamList_, and verbosity_.

Referenced by analyze(), and restoreStreamsOfLumiBlock().

                                                            {

  if((fedId != -1) &&
     (fedId < ecalDccFedIdMin_ || fedId > ecalDccFedIdMax_)) fedId = -1;
  
  if(verbosity_>1) cout << "[LaserSorter " << now() << "] "
       << "Looking for an opened output file for FED " 
       << fedId << " LB " << lumiBlock
       << "\n";

  //first look if stream is already open:
  for(OutStreamList::iterator it = outStreamList_.begin();
      it != outStreamList_.end();
      ++it){
    if(it->fedId()==fedId && 
       (abs((int)it->startingLumiBlock()-(int)lumiBlock)<=lumiBlockSpan_)){
      //stream found!
      return &(*it);
    }
  }
  //stream was not found. Let's create one

  if(verbosity_) cout << "[LaserSorter " << now() << "] "
                      << "File not yet opened. Opening it.\n";

  OutStreamList::iterator streamRecord = createOutStream(fedId, lumiBlock);
  return streamRecord!=outStreamList_.end()?&(*streamRecord):0;
}

bool LaserSorter::isDccEventEmpty ( const FEDRawData &  data, size_t *  dccLen = 0, int *  nTowerBlocks = 0  ) const

private

Checks if an ECAL DCC event is empty. It is considered as empty if it does not contains FE data ("tower" block). So an event containing SRP or TCC data can be tagged as empty by this method. , if not null filled with the event length read from the DCC header. if not null, filled with number of tower blocks

Returns

true if event is empty, false otherwise

Definition at line 954 of file LaserSorter.cc.

References FEDRawData::data(), dtTPAnalyzer_cfg::dataType, and FEDRawData::size().

Referenced by getFullyReadoutDccs(), and getOutputFedList().

                                                          {
  if(nTowerBlocks) *nTowerBlocks = 0;
  //DCC event is considered empty if it does not contains any Tower block
  //( = FE data)
  bool rc = true;
  if(dccLen) *dccLen = 0;
  const unsigned nWord32 = data.size()/sizeof(uint32_t);
  if(nWord32==0){
    //cout << "[LaserSorter " << now() << "] " << "FED block completly empty\n";
    return true;
  }
  for(unsigned iWord32 = 0; iWord32 < nWord32; iWord32 += 2){
    const uint32_t* data32 = ((const uint32_t*)(data.data())) + iWord32;
    int dataType = (data32[1] >>28) & 0xF;
    //     cout << hex << "0x" << setfill('0')
    //          << setw(8) << data32[1] << "'" << setw(8) << data32[0]
    //          << " dataType: 0x" << dataType 
    //          << dec << setfill(' ') << "\n";
    if(0==(dataType>>2)){//in DCC header
      const int dccHeaderId = (data32[1] >>24) & 0x3F;
      if(dccHeaderId==1){
        if(dccLen) *dccLen=((data32[0] >>0 ) & 0xFFFFFF);
      }
    } if((dataType>>2)==3){//Tower block
      rc = false;
      if(nTowerBlocks){//number of tower block must be counted
        ++(*nTowerBlocks);
      } else{
        break;
      }
    }
  }
  //   cout << "[LaserSorter " << now() << "] " << "DCC Len: ";

  //   if(dccLen){
  //     cout << (*dccLen) << " event ";
  //   }
  //   cout << (rc?"":"non") << " empty"
  //        << endl;
  return rc;
}

int LaserSorter::readFormatVersion ( std::ifstream &  in, const std::string &  fileName  )

private

Gets format version of an LMF file. Position of file is preserved.

Parameters

in	stream to read the file
fileName	name of the file. Used in error message.

Returns

version or -1 in case of error.

Definition at line 1151 of file LaserSorter.cc.

References data, AlCaHLTBitMon_ParallelJobs::p, and AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by createOutStream().

                                                             {
  int vers = -1;
  streampos p = in.tellg();

  uint32_t data;

  in.read((char*)&data, sizeof(data));
  
  char magic[4];
  
  magic[0] = data & 0xFF;
  magic[1] = (data >>8) & 0xFF;
  magic[2] = (data >>16) & 0xFF;
  magic[3] = 0;


  const string lmf = string("LMF");
  
  if(in.good() && lmf.compare(magic)==0){
    vers = (data >>24) & 0xFF;
  }

  if(lmf.compare(magic)!=0){
    edm::LogWarning("LaserSorter") << "File " << fileName
                                   << "is not an LMF file.\n";
  }
  
  in.clear();
  in.seekg(p);
  return vers;
}

bool LaserSorter::readIndexTable ( std::ifstream &  in, std::string &  inName, OutStreamRecord &  outRcd, std::string *  err  )

private

Read index table of an LMF file.

Parameters

in	LMF file whose index table must be read.
inName	name of the in file
outRcd	record of the output file. whose the index table must be copied to.
err	if not nul, in case of failure filled with the error message.

Returns

true in case of success, false otherwise

Definition at line 1077 of file LaserSorter.cc.

References gather_cfg::cout, LaserSorter::OutStreamRecord::excludedOrbit(), i, recoMuon::in, LaserSorter::OutStreamRecord::indices(), maxEvents_, jetmet_cfg::nevts, now(), alignCSCRings::s, and verbosity_.

Referenced by createOutStream().

                                                {
  stringstream errMsg;

  ifstream* s = ∈
  
  //streampos pos = s->tellg();
  s->clear();
  s->seekg(0);
  
  uint32_t fileHeader[2];
  s->read((char*)&fileHeader[0], sizeof(fileHeader));
  uint32_t indexTablePos = fileHeader[1]; 
  
  if(s->eof()){
    s->clear();
    s->seekg(0);
    errMsg << "Failed to read header of file " << inName
           << ".";
    if(err) *err = errMsg.str();
    return false;
  }
  
  s->seekg(indexTablePos);

  uint32_t nevts = 0;
  s->read((char*)&nevts, sizeof(nevts));
  s->ignore(4);  
  if(s->bad()){
    errMsg << "Failed to read index table from file "
           << inName << ".";
    if(err) *err = errMsg.str();
    return false;
  }
  if(nevts>maxEvents_){
    errMsg << "Number of events indicated in event index of file "
           << inName << " (" << nevts << ") "
           << "is unexpectively large.";
    if(err) *err = errMsg.str();
    return false;
  }
  outRcd.indices()->resize(nevts);
  s->read((char*)&(*outRcd.indices())[0], nevts*sizeof(IndexRecord));
  if(s->bad()){
    outRcd.indices()->clear();
    errMsg << "Failed to read index table from file "
           << inName << ".";
    if(err) *err = errMsg.str();
    return false;
  }
  if(nevts>maxEvents_){
    errMsg << "Number of events indicated in event index of file "
           << inName << " is unexpectively large.";
    if(err) *err = errMsg.str();
    outRcd.indices()->clear();
    return false;
  }

  if(verbosity_ > 1) cout << "[LaserSorter " << now() << "] " << "Orbit IDs of events "
                          << "already contained in the file "
                          << inName << ":";
  for(unsigned i = 0; i < outRcd.indices()->size(); ++i){
    if(verbosity_>1){
      cout << " "  << setw(9) << (*outRcd.indices())[i].orbit;
    }
    outRcd.excludedOrbit().insert((*outRcd.indices())[i].orbit);
  }
  if(verbosity_>1) cout << "\n";

  return true;
}

bool LaserSorter::renameAsBackup ( const std::string &  fileName, std::string &  newFileName  )

private

Definition at line 618 of file LaserSorter.cc.

References i, and MainPageGenerator::link.

Referenced by createOutStream().

                                                        {
  int i = 0;
  int err;
  //  static int maxTries = 100;
  int maxTries = 20;
  stringstream newFileName_;
  do{
    newFileName_.str("");
    newFileName_ << fileName << "~";
    if(i>0) newFileName_ << i;
    err = link(fileName.c_str(), newFileName_.str().c_str());
    if(err==0){
      newFileName = newFileName_.str();
      err = unlink(fileName.c_str());
    }
    ++i;
  } while((err!=0) && (errno == EEXIST) && (i < maxTries));
  return err==0;
}

void LaserSorter::restoreStreamsOfLumiBlock ( int  lumiBlock )

private

Opens output streams associated to a lumi block according to already existing files. To be used when previously processed luminosity block is not

Parameters

lumiBlock

ID of the luminosity block whose output streams must be reopened.

Definition at line 1199 of file LaserSorter.cc.

References ecalDccFedIdMax_, ecalDccFedIdMin_, HLT_25ns14e33_v1_cff::fedId, convertXMLtoSQLite_cfg::fileName, getStream(), alignCSCRings::s, and streamFileName().

Referenced by analyze().

                                                        {
  string dummy;
  string fileName;
  
  for(int fedId = ecalDccFedIdMin_-2; fedId <= ecalDccFedIdMax_; ++fedId){
    int fedId_;
    if(fedId == ecalDccFedIdMin_-2) fedId_ = -1; //stream for event w/o ECAL data
    else fedId_ = fedId;
    streamFileName(fedId_, lumiBlock, dummy, fileName);
    struct stat s;
    //TODO: could be optimized by adding an option to get stream
    //to open only existing file: would avoid double call to streamFileName.
    if(stat(fileName.c_str(), &s)==0){//file exists
      getStream(fedId_, lumiBlock);
    }
  }
}

void LaserSorter::streamFileName ( int  fedId, edm::LuminosityBlockNumber_t  lumiBlock, std::string &  tmpName, std::string &  finalName  )

private

Builds the file names for the group of event corresponding to a FED and a starting lumi block.

Parameters

	fedId	FED ID of the event set
	lumiBlock	starting luminoisty block of the event set
[out]	tmpName	name of the file to use when filling it.
[out]	finalName	name of the file once completed.

Definition at line 843 of file LaserSorter.cc.

References gather_cfg::cout, dir, ecalDccFedIdMax_, ecalDccFedIdMin_, edm::hlt::Exception, fedSubDirs_, convertXMLtoSQLite_cfg::fileName, now(), outputDir_, runNumber_, and verbosity_.

Referenced by createOutStream(), and restoreStreamsOfLumiBlock().

                                                                          {
  int iFed;
  if(fedId >= ecalDccFedIdMin_ && fedId <= ecalDccFedIdMax_){
    iFed = fedId - ecalDccFedIdMin_ + 1;
  } else if(fedId < 0){
    iFed = -1; //event w/o ECAL data
  } else {
    iFed = 0;
  }
  if(iFed < -1 || iFed >= (int)fedSubDirs_.size()){
    throw cms::Exception("LaserSorter")
      << "Bug found at " << __FILE__ << ":" << __LINE__ 
      << ". FED ID is out of index!";
  }

  
  struct stat fileStat;

  stringstream buf;
  buf << outputDir_ << "/"
      << (iFed<0 ? "Empty" : fedSubDirs_[iFed]);

  string dir = buf.str();
  if(0==stat(dir.c_str(), &fileStat)){
    if(!S_ISDIR(fileStat.st_mode)){
      throw cms::Exception("[LaserSorter]")
        << "File " << dir << " exists but is not a directory "
        << " as expected.";
    }
  } else {//directory does not exists, let's try to create it
    if(0!=mkdir(dir.c_str(), 0755)){
      throw cms::Exception("[LaserSorter]")
        << "Failed to create directory " << dir
        << " for writing data.";
    }
  }

  buf.str("");
  buf << "Run" << runNumber_
      << "_LB" << setfill('0') << setw(4) << lumiBlock
      << ".lmf";
  string fileName = buf.str();
  string tmpFileName = fileName + ".part";

  finalName = dir + "/" + fileName;
  tmpName = dir + "/" + tmpFileName;

  if(verbosity_>3) cout << "[LaserSorter " << now() << "] " << "File path: "
                        << finalName << "\n";
}

std::string LaserSorter::toString ( uint64_t  t )

staticprivate

Help function to format a date

Definition at line 1184 of file LaserSorter.cc.

References AlCaHLTBitMon_QueryRunRegistry::string.

Referenced by writeEventHeader().

                                         {
  char buf[256];
  
  time_t tsec = t>>32;
  
  uint32_t tusec = t & 0xFFFFFFFF;
  strftime(buf, sizeof(buf), "%F %R %S s", localtime(&tsec));
  buf[sizeof(buf)-1] = 0;
  
  stringstream buf2;
  buf2 << (tusec+500)/1000;

  return string(buf) + " " + buf2.str() + " ms";
}

bool LaserSorter::writeEvent ( OutStreamRecord &  out, const edm::Event &  event, int  detailedTriggerType, const FEDRawDataCollection &  data  )

private

Writes a monitoring events to an output stream.

Parameters

out	stream to write the event out
event	EDM event, used to retrieve meta information like timestamp and ID.
ID	of the unique FED block of the event
data	DCC data

Returns

true on success, false on failure

See Also

getStream(int, edm::LuminosityBlockNumber_t)

Definition at line 530 of file LaserSorter.cc.

References gather_cfg::cout, FEDRawData::data(), dt, FEDRawDataCollection::FEDData(), getOutputFedList(), MatacqRawEvent::getTimeStamp(), LaserSorter::OutStreamRecord::indices(), matacqFedId_, now(), orbit_, orbitZeroTime_, LaserSorter::OutStreamRecord::out(), dbtoconf::out, FEDRawData::size(), edm::Event::size(), verbosity_, writeEventHeader(), and writeFedBlock().

Referenced by analyze().

                                                              {

  ofstream& out = *outRcd.out();
  bool rc = true;
  vector<unsigned> fedIds;
  getOutputFedList(event, data, fedIds);
  
  out.clear();
  uint32_t evtStart = out.tellp();
  if(out.bad()) evtStart = 0;
  rc &= writeEventHeader(out, event, dtt, fedIds.size());
  
  if(orbitZeroTime_.tv_sec==0 && data.FEDData(matacqFedId_).size()!=0){
    struct timeval ts = {0, 0};
    MatacqRawEvent mre(data.FEDData(matacqFedId_).data(), data.FEDData(matacqFedId_).size());
    mre.getTimeStamp(ts);
    uint32_t orb = mre.getOrbitId();
      if(ts.tv_sec!=0){
        div_t dt = div(orb * 89.1, 1000*1000); //an orbit lasts 89.1 microseconds
        orbitZeroTime_.tv_sec = ts.tv_sec - dt.quot;
        orbitZeroTime_.tv_usec = ts.tv_usec - dt.rem;
        if(orbitZeroTime_.tv_usec < 0){
          orbitZeroTime_.tv_usec += 1000*1000;
          orbitZeroTime_.tv_sec -= 1;
        }
      }
  }

  for(unsigned iFed = 0; iFed < fedIds.size() && rc; ++iFed){
    if(verbosity_>3) cout << "[LaserSorter " << now() << "] " 
                          << "Writing data block of FED " << fedIds[iFed] << ". Data size: "  
                          <<  data.FEDData(fedIds[iFed]).size() << "\n"; 
    rc  &= writeFedBlock(out, data.FEDData(fedIds[iFed]));
  }

  if(rc){
    //update index table for this file:
    vector<IndexRecord>& indices = *outRcd.indices();
    if(verbosity_ > 2){
      std::cout << "Event " << " written successfully. "
                << "Orbit: " << orbit_
                << "\tFile index: " << evtStart << "\n";
    }
    IndexRecord indexRcd = {(uint32_t)orbit_, evtStart};
    indices.push_back(indexRcd);
  }
  return rc;
}

bool LaserSorter::writeEventHeader ( std::ofstream &  out, const edm::Event &  evt, int  fedId, unsigned  nFeds  )

private

Write event header with event identification and timestamp.

Parameters

out	output stream to write to
evt	event

Returns

false in case of write failure

Definition at line 793 of file LaserSorter.cc.

References edm::EventBase::bunchCrossing(), colorNames, gather_cfg::cout, TauDecayModes::dec, detailedTrigNames, dt, edm::EventID::event(), edm::EventBase::id(), edm::EventBase::luminosityBlock(), now(), orbit_, orbitZeroTime_, edm::Event::run(), edm::EventBase::time(), toString(), groupFilesInBlocks::tt, edm::Timestamp::value(), and verbosity_.

Referenced by writeEvent().

                                                  {
  uint32_t data[10];
  timeval tt = {0, 0};
  if((evt.time().value() >>32)){
    tt.tv_usec = evt.time().value() & 0xFFFFFFFF;
    tt.tv_sec = evt.time().value() >>32;
  } else if(orbitZeroTime_.tv_sec){
    div_t dt = div(orbit_*89.1, 1000*1000); //one orbit lasts 89.1 microseconds
    tt.tv_sec = orbitZeroTime_.tv_sec + dt.quot;
    tt.tv_usec = orbitZeroTime_.tv_usec + dt.rem;
    if(tt.tv_usec > 1000*1000){
      tt.tv_usec -= 1000*1000;
      tt.tv_sec += 1;
    }
  }
    
  data[0] = tt.tv_usec;
  data[1] = tt.tv_sec;
  data[2] = evt.luminosityBlock();
  data[3] = evt.run();     
  data[4] = orbit_;
  data[5] = evt.bunchCrossing();
  data[6] = evt.id().event();
  data[7] = dtt;
  data[8] = nFeds;
  data[9] = 0; //reserved (to be aligned on 64-bits)

  if(verbosity_>1){
    cout << "[LaserSorter " << now() << "] " << "Write header of event: "
         << "Time: " << toString(evt.time().value())
         << ", LB: " << evt.luminosityBlock()
         << ", Run: " << evt.run()
         << ", Bx: " << evt.bunchCrossing()
         << ", Event ID: " << evt.id().event()
         << ", Detailed trigger type: 0x" << hex << dtt << dec
         << " (" << detailedTrigNames[(dtt>>8)&0x7]  << ", "
         << colorNames[(dtt>>6)&0x3] << ", DCC " << (dtt&0x3f)
         << ", side " << ((dtt>>10) &0x1) << ")"
         << ", number of FEDs: "
         << "\n";
  }
    
  out.clear();
  out.write((char*)data, sizeof(data));
  return !out.bad();
}

bool LaserSorter::writeFedBlock ( std::ofstream &  out, const FEDRawData &  data  )

private

Writes out data of a FED

Parameters

out	stream to write the event out
data	FED data

Returns

true on success, false on failure

Definition at line 581 of file LaserSorter.cc.

References gather_cfg::cout, FEDRawData::data(), edm::hlt::Exception, now(), FEDRawData::size(), and verbosity_.

Referenced by writeEvent().

                                                       {
  bool rc = false;
  if (data.size()>4){
    const uint32_t * pData
      = reinterpret_cast<const uint32_t*>(data.data());
    
    uint32_t dccLen64 = pData[2] & 0x00FFFFFF; //in 32-byte unit

    if(data.size() != dccLen64*sizeof(uint64_t)){
//       throw cms::Exception("Bug") << "Bug found in "
//                                   << __FILE__ << ":" << __LINE__ << ".";
      throw cms::Exception("LaserSorter")
        << "Mismatch between FED fragment size indicated in header "
        << "(" << dccLen64 << "*8 Byte) "
        << "and actual size (" <<  data.size() << " Byte) "
        << "for FED ID " <<  ((pData[0] >>8) & 0xFFF) << "!\n";
    }
    
    if(verbosity_>3) cout << "[LaserSorter " << now() << "] " << "Event fragment size: "
                          << data.size() << " Byte"
                          << "\t From Dcc header: " << dccLen64*8 << " Byte\n";
    
    const size_t nBytes = data.size();
    //       cout << "[LaserSorter " << now() << "] " 
    //            << "Writing " << nBytes << " byte from adress " 
    //            << (void*) data.data() << " to file.\n";
    if(out.fail()) cout << "[LaserSorter " << now() << "] " << "Problem with stream!\n";
    out.write((char*)data.data(), nBytes);
    rc = true;
  } else{
    throw cms::Exception("Bug") << "Bug found in "
                                << __FILE__ << ":" << __LINE__ << ".\n";
  }
  return rc;
}

void LaserSorter::writeFileHeader ( std::ofstream &  out )

private

Writing file header for an LMF binary file

Parameters

out	stream of the output file

Definition at line 776 of file LaserSorter.cc.

References edm::hlt::Exception, and formatVersion_.

Referenced by createOutStream().

                                                 {
  out.clear();
  
  uint32_t id = 'L' | ('M' <<8) | ('F' <<16) | (formatVersion_<<24);

  out.write((char*)&id, sizeof(uint32_t));

  //index position (to be filled at end of writing)
  uint32_t zero = 0;
  out.write((char*)&zero, sizeof(uint32_t));

  if(out.fail()){
    throw cms::Exception("LaserSorter")
      << "Failed to write file header.\n";
  }
}

bool LaserSorter::writeIndexTable ( std::ofstream &  out, std::vector< IndexRecord > &  indices  )

private

Writes index table in LMF output file. stream must be positionned to the place for the index table (end of file).

Parameters

out	stream of output file.
indices	index table

Definition at line 1037 of file LaserSorter.cc.

References end, i, indexOffset32_, jetmet_cfg::nevts, and python.multivaluedict::sort().

Referenced by closeOutStream().

                                                                  {
  uint32_t indexTablePos = out.tellp();
  uint32_t nevts = indices.size();
  
  out.clear();
  out.write((char*)&nevts, sizeof(nevts));
  const uint32_t reserved = 0;
  out.write((const char*)&reserved, sizeof(reserved));
  
  if(out.bad()) return false;

  sort(indices.begin(), indices.end());

  for(unsigned i = 0; i < indices.size(); ++i){
    uint32_t data[2];
    data[0] = indices[i].orbit;
    data[1] = indices[i].filePos;
    out.write((char*)data, sizeof(data));
  }

  if(out.bad()) return false; //intial 0 valur for index table position
  //                            is left to indicate corrupted table.
  
  //writes index table position:x
  out.clear();
  out.seekp(indexOffset32_*sizeof(uint32_t));
  //   cout << "[LaserSorter] Index table position: 0x" << hex << indexTablePos
  //        << dec << "\n"; 
  if(!out.bad()) out.write((char*)&indexTablePos, sizeof(uint32_t));

  bool rc = !out.bad();
  
  //reposition pointer to eof:
  out.seekp(0, ios::end);

  return rc;
}

Member Data Documentation

int LaserSorter::detailedTrigType_

private

Trigger type of calibration event. -1 if unkown.

Definition at line 320 of file LaserSorter.h.

Referenced by analyze().

bool LaserSorter::disableOutput_

private

Switch to disable writing output file (for test purpose).

Definition at line 357 of file LaserSorter.h.

Referenced by analyze().

bool LaserSorter::doOutputList_

private

Switch for logging paths of the output files

Definition at line 381 of file LaserSorter.h.

Referenced by closeOutStream(), and LaserSorter().

const int LaserSorter::ecalDccFedIdMax_ = 654

staticprivate

Lower bound of ECAL DCC FED ID

Definition at line 316 of file LaserSorter.h.

Referenced by analyze(), getDetailedTriggerType(), getFullyReadoutDccs(), getOrbitFromDcc(), getOutputFedList(), getStream(), restoreStreamsOfLumiBlock(), and streamFileName().

const int LaserSorter::ecalDccFedIdMin_ = 601

staticprivate

Lower bound of ECAL DCC FED ID

Definition at line 312 of file LaserSorter.h.

Referenced by analyze(), getDetailedTriggerType(), getFullyReadoutDccs(), getOrbitFromDcc(), getOutputFedList(), restoreStreamsOfLumiBlock(), and streamFileName().

edm::InputTag LaserSorter::fedRawDataCollectionTag_

private

Definition at line 441 of file LaserSorter.h.

Referenced by LaserSorter().

edm::EDGetTokenT<FEDRawDataCollection> LaserSorter::fedRawDataCollectionToken_

private

Definition at line 442 of file LaserSorter.h.

Referenced by analyze(), and LaserSorter().

std::vector<std::string> LaserSorter::fedSubDirs_

private

Subdirectories for output file of each FED

Definition at line 349 of file LaserSorter.h.

Referenced by LaserSorter(), and streamFileName().

unsigned char LaserSorter::formatVersion_

private

Data format version of lmf output file

Definition at line 341 of file LaserSorter.h.

Referenced by createOutStream(), and writeFileHeader().

const int LaserSorter::indexOffset32_ = 1

staticprivate

Definition at line 450 of file LaserSorter.h.

Referenced by writeIndexTable().

int LaserSorter::iNoEcalDataMess_

private

number of "ECAL DCC data" message in a run

Definition at line 422 of file LaserSorter.h.

Referenced by analyze(), and getDetailedTriggerType().

int LaserSorter::iNoFullReadoutDccError_

private

Number of "No fully readout DCC error"

Definition at line 412 of file LaserSorter.h.

Referenced by analyze().

std::ofstream LaserSorter::logFile_

private

File for logging

Definition at line 324 of file LaserSorter.h.

Referenced by analyze(), LaserSorter(), and ~LaserSorter().

edm::LuminosityBlockNumber_t LaserSorter::lumiBlock_

private

Luminosity block of event under processing

Definition at line 328 of file LaserSorter.h.

Referenced by analyze().

edm::LuminosityBlockNumber_t LaserSorter::lumiBlockPrev_

private

Luminosity block of previous processed event

Definition at line 332 of file LaserSorter.h.

Referenced by analyze().

int LaserSorter::lumiBlockSpan_

private

Tolerance on lumi block spanning of a FED sequence. Subsequent events of a same FED must span at most on 2*lumiBlockSpan_+1 luminosity blocks.

• It is important that the laser sequence scane does not pass twice on the same FED within the 2*lumiBlockSpan_+1. Failing this requirement will result mixing event of different passes in the same output file.
• The number of input files opened simultinuously is proportional to 2*lumiBlockSpan_+1. So increasing lumiBlockSpan_ will also increase the number of opened files and may have some impact of sorting time performances.

Definition at line 439 of file LaserSorter.h.

Referenced by analyze(), closeOldStreams(), and getStream().

const int LaserSorter::matacqFedId_ = 655

staticprivate

FED ID associated to Matacq data

Definition at line 446 of file LaserSorter.h.

Referenced by getOutputFedList(), and writeEvent().

const unsigned LaserSorter::maxEvents_ = 1<<20

staticprivate

Limit of number of events to prevent exhausting memory with indexTable_ in case of file corruption.

Definition at line 455 of file LaserSorter.h.

Referenced by readIndexTable().

int LaserSorter::maxFullReadoutDccError_

private

Maximum number of "No fully readout DCC error" message in a run

Definition at line 417 of file LaserSorter.h.

Referenced by analyze().

int LaserSorter::maxNoEcalDataMess_

private

Maximum number of "ECAL DCC data" message in a run

Definition at line 426 of file LaserSorter.h.

Referenced by getDetailedTriggerType().

int LaserSorter::orbit_

private

Definition at line 486 of file LaserSorter.h.

Referenced by analyze(), writeEvent(), and writeEventHeader().

int LaserSorter::orbitCountInALumiBlock_

private

Length of a lumi block in number of orbits used when overWriteLumiBlockId is set to true; LB = orbit_id / orbitCountInALumiBlock_

Definition at line 484 of file LaserSorter.h.

Referenced by analyze().

struct timeval LaserSorter::orbitZeroTime_

private

Time stamp offset to use to calculate calibration time from the orbit id. It is obtained from the matacq block (time originates from the PC hosting the laser supervisor). and used in case the event is missing timestamp, in principle for data acquired with the minidaq Calibration event time = orbitZeroTime_ + orbit_ * (89.1 microsec).

Definition at line 495 of file LaserSorter.h.

Referenced by writeEvent(), and writeEventHeader().

std::string LaserSorter::outputDir_

private

Top directory for output files

Definition at line 345 of file LaserSorter.h.

Referenced by LaserSorter(), batchmanager.BatchManager::PrepareJob(), valtools.webpage::readCaptions(), and streamFileName().

std::ofstream LaserSorter::outputList_

private

stream where list of output file is listed to

Definition at line 389 of file LaserSorter.h.

Referenced by closeOutStream(), and LaserSorter().

std::string LaserSorter::outputListFile_

private

name of file where list of output file is listed to

Definition at line 377 of file LaserSorter.h.

Referenced by LaserSorter().

OutStreamList LaserSorter::outStreamList_

private

List of output stream to write sorted data

Definition at line 337 of file LaserSorter.h.

Referenced by closeAllStreams(), closeOldStreams(), closeOutStream(), createOutStream(), and getStream().

bool LaserSorter::overWriteLumiBlockId_

private

Switch to recompute and overwrite the lumi block ID

Definition at line 477 of file LaserSorter.h.

Referenced by analyze().

edm::RunNumber_t LaserSorter::runNumber_

private

Run number of event under process

Definition at line 361 of file LaserSorter.h.

Referenced by analyze(), and streamFileName().

struct LaserSorter::stats_t LaserSorter::stats_

private

Referenced by analyze(), and ~LaserSorter().

const LaserSorter::stats_t LaserSorter::stats_init = {0, 0, 0, 0, 0}

staticprivate

Definition at line 473 of file LaserSorter.h.

std::ofstream LaserSorter::timeLog_

private

Output stream to log code timing

Definition at line 369 of file LaserSorter.h.

Referenced by analyze(), and LaserSorter().

std::string LaserSorter::timeLogFile_

private

Name of the file to log the processing time

Definition at line 353 of file LaserSorter.h.

Referenced by LaserSorter().

timeval LaserSorter::timer_

private

Buffer for timing

Definition at line 365 of file LaserSorter.h.

Referenced by analyze(), and LaserSorter().

bool LaserSorter::timing_

private

Switch for code timing.

Definition at line 373 of file LaserSorter.h.

Referenced by analyze(), and LaserSorter().

int LaserSorter::verbosity_

private

Debugging message verbosity level

Definition at line 385 of file LaserSorter.h.

Referenced by analyze(), closeOldStreams(), closeOutStream(), createOutStream(), getDetailedTriggerType(), getStream(), readIndexTable(), streamFileName(), writeEvent(), writeEventHeader(), and writeFedBlock().