CMS 3D CMS Logo

/afs/cern.ch/work/a/aaltunda/public/www/CMSSW_6_2_5/src/DQMServices/FwkIO/plugins/DQMRootSource.cc

Go to the documentation of this file.
00001 // -*- C++ -*-
00002 //
00003 // Package:     FwkIO
00004 // Class  :     DQMRootSource
00005 // 
00006 // Implementation:
00007 //     [Notes on implementation]
00008 //
00009 // Original Author:  Chris Jones
00010 //         Created:  Tue May  3 11:13:47 CDT 2011
00011 // $Id: DQMRootSource.cc,v 1.32 2013/06/06 07:52:17 deguio Exp $
00012 //
00013 
00014 // system include files
00015 #include <vector>
00016 #include <string>
00017 #include <map>
00018 #include <memory>
00019 #include <list>
00020 #include <set>
00021 #include "TFile.h"
00022 #include "TTree.h"
00023 #include "TString.h"
00024 #include "TH1.h"
00025 #include "TH2.h"
00026 #include "TProfile.h"
00027 
00028 // user include files
00029 #include "FWCore/Framework/interface/InputSource.h"
00030 #include "FWCore/Catalog/interface/InputFileCatalog.h"
00031 #include "FWCore/ParameterSet/interface/ParameterSet.h"
00032 #include "DQMServices/Core/interface/DQMStore.h"
00033 #include "DQMServices/Core/interface/MonitorElement.h"
00034 #include "FWCore/ServiceRegistry/interface/Service.h"
00035 #include "FWCore/MessageLogger/interface/JobReport.h"
00036 //#include "FWCore/Utilities/interface/GlobalIdentifier.h"
00037 #include "FWCore/Utilities/interface/EDMException.h"
00038 
00039 #include "FWCore/Framework/interface/RunPrincipal.h"
00040 #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h"
00041 
00042 #include "FWCore/Framework/interface/Run.h"
00043 #include "FWCore/Framework/interface/LuminosityBlock.h"
00044 
00045 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
00046 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
00047 
00048 #include "FWCore/Framework/interface/InputSourceMacros.h"
00049 #include "FWCore/Framework/interface/FileBlock.h"
00050 
00051 #include "FWCore/MessageLogger/interface/MessageLogger.h"
00052 #include "FWCore/Utilities/interface/TimeOfDay.h"
00053 
00054 #include "DataFormats/Provenance/interface/ProcessHistory.h"
00055 #include "DataFormats/Provenance/interface/ProcessHistoryID.h"
00056 #include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h"
00057 #include "FWCore/ParameterSet/interface/Registry.h"
00058 
00059 #include "FWCore/Utilities/interface/Digest.h"
00060 
00061 #include "format.h"
00062 
00063 namespace {
00064   //adapter functions
00065   MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1F* iHist) {
00066     //std::cout <<"create: hist size "<<iName <<" "<<iHist->GetEffectiveEntries()<<std::endl;
00067     return iStore.book1D(iName, iHist);
00068   }
00069   //NOTE: the merge logic comes from DataFormats/Histograms/interface/MEtoEDMFormat.h
00070   void mergeTogether(TH1* iOriginal,TH1* iToAdd) {
00071     if(iOriginal->TestBit(TH1::kCanRebin)==true && iToAdd->TestBit(TH1::kCanRebin) ==true) {
00072       TList list;
00073       list.Add(iToAdd);
00074       if( -1 == iOriginal->Merge(&list)) {
00075         edm::LogError("MergeFailure")<<"Failed to merge DQM element "<<iOriginal->GetName();
00076       }
00077     } else {
00078       if (iOriginal->GetNbinsX() == iToAdd->GetNbinsX() &&
00079           iOriginal->GetXaxis()->GetXmin() == iToAdd->GetXaxis()->GetXmin() &&
00080           iOriginal->GetXaxis()->GetXmax() == iToAdd->GetXaxis()->GetXmax() &&
00081           iOriginal->GetNbinsY() == iToAdd->GetNbinsY() &&
00082           iOriginal->GetYaxis()->GetXmin() == iToAdd->GetYaxis()->GetXmin() &&
00083           iOriginal->GetYaxis()->GetXmax() == iToAdd->GetYaxis()->GetXmax() &&
00084           iOriginal->GetNbinsZ() == iToAdd->GetNbinsZ() &&
00085           iOriginal->GetZaxis()->GetXmin() == iToAdd->GetZaxis()->GetXmin() &&
00086           iOriginal->GetZaxis()->GetXmax() == iToAdd->GetZaxis()->GetXmax()) {
00087         iOriginal->Add(iToAdd);
00088       } else {
00089         edm::LogError("MergeFailure")<<"Found histograms with different axis limits '"<<iOriginal->GetName()<<"' not merged.";
00090       }
00091     } 
00092   }
00093   void mergeWithElement(MonitorElement* iElement, TH1F* iHist) {
00094     //std::cout <<"merge: hist size "<<iElement->getName() <<" "<<iHist->GetEffectiveEntries()<<std::endl;
00095     mergeTogether(iElement->getTH1F(),iHist);
00096   }
00097   MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1S* iHist) {
00098     return iStore.book1S(iName, iHist);
00099   }
00100   void mergeWithElement(MonitorElement* iElement, TH1S* iHist) {
00101     mergeTogether(iElement->getTH1S(),iHist);
00102   }  
00103   MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1D* iHist) {
00104     return iStore.book1DD(iName, iHist);
00105   }
00106   void mergeWithElement(MonitorElement* iElement, TH1D* iHist) {
00107     mergeTogether(iElement->getTH1D(),iHist);
00108   }
00109   MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2F* iHist) {
00110     return iStore.book2D(iName, iHist);
00111   }
00112   void mergeWithElement(MonitorElement* iElement, TH2F* iHist) {
00113     mergeTogether(iElement->getTH2F(),iHist);
00114   }
00115   MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2S* iHist) {
00116     return iStore.book2S(iName, iHist);
00117   }
00118   void mergeWithElement(MonitorElement* iElement, TH2S* iHist) {
00119     mergeTogether(iElement->getTH2S(),iHist);
00120   }  
00121   MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2D* iHist) {
00122     return iStore.book2DD(iName, iHist);
00123   }
00124   void mergeWithElement(MonitorElement* iElement, TH2D* iHist) {
00125     mergeTogether(iElement->getTH2D(),iHist);
00126   }
00127   MonitorElement* createElement(DQMStore& iStore, const char* iName, TH3F* iHist) {
00128     return iStore.book3D(iName, iHist);
00129   }
00130   void mergeWithElement(MonitorElement* iElement, TH3F* iHist) {
00131     mergeTogether(iElement->getTH3F(),iHist);
00132   }
00133   MonitorElement* createElement(DQMStore& iStore, const char* iName, TProfile* iHist) {
00134     return iStore.bookProfile(iName, iHist);
00135   }
00136   void mergeWithElement(MonitorElement* iElement, TProfile* iHist) {
00137     mergeTogether(iElement->getTProfile(),iHist);
00138   }
00139   MonitorElement* createElement(DQMStore& iStore, const char* iName, TProfile2D* iHist) {
00140     return iStore.bookProfile2D(iName, iHist);
00141   }
00142   void mergeWithElement(MonitorElement* iElement, TProfile2D* iHist) {
00143     mergeTogether(iElement->getTProfile2D(),iHist);
00144   }
00145 
00146   MonitorElement* createElement(DQMStore& iStore, const char* iName, Long64_t& iValue) {
00147     MonitorElement* e = iStore.bookInt(iName);
00148     e->Fill(iValue);
00149     return e;
00150   }
00151 
00152   //NOTE: the merge logic comes from DataFormats/Histograms/interface/MEtoEDMFormat.h
00153   void mergeWithElement(MonitorElement* iElement, Long64_t& iValue) {
00154     const std::string& name = iElement->getFullname();
00155     if(name.find("EventInfo/processedEvents") != std::string::npos) {
00156       iElement->Fill(iValue+iElement->getIntValue());
00157     } else {
00158       if(name.find("EventInfo/iEvent") != std::string::npos ||
00159          name.find("EventInfo/iLumiSection") != std::string::npos) {
00160         if(iValue > iElement->getIntValue()) {
00161           iElement->Fill(iValue);
00162         }
00163       }
00164     }
00165   }
00166 
00167   MonitorElement* createElement(DQMStore& iStore, const char* iName, double& iValue) {
00168     MonitorElement* e = iStore.bookFloat(iName);
00169     e->Fill(iValue);
00170     return e;
00171   }
00172   void mergeWithElement(MonitorElement* iElement, double& iValue) {
00173     //no merging
00174   }
00175   MonitorElement* createElement(DQMStore& iStore, const char* iName, std::string* iValue) {
00176     return iStore.bookString(iName,*iValue);
00177   }
00178   void mergeWithElement(MonitorElement* , std::string* ) {
00179     //no merging
00180   }
00181 
00182   void splitName(const std::string& iFullName, std::string& oPath,const char*& oName) {
00183     oPath = iFullName;
00184     size_t index = oPath.find_last_of('/');
00185     if(index == std::string::npos) {
00186       oPath = std::string();
00187       oName = iFullName.c_str();
00188     } else {
00189       oPath.resize(index);
00190       oName = iFullName.c_str()+index+1;
00191     }
00192   }
00193 
00194   struct RunLumiToRange {
00195     unsigned int m_run, m_lumi,m_historyIDIndex;
00196     ULong64_t m_beginTime;
00197     ULong64_t m_endTime;
00198     ULong64_t m_firstIndex, m_lastIndex; //last is inclusive
00199     unsigned int m_type; //A value in TypeIndex
00200   };
00201 
00202   class TreeReaderBase {
00203     public:
00204       TreeReaderBase() {}
00205       virtual ~TreeReaderBase() {}
00206 
00207       MonitorElement* read(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi){
00208         return doRead(iIndex,iStore,iIsLumi);
00209       }
00210       virtual void setTree(TTree* iTree) =0;
00211     protected:
00212       TTree* m_tree;
00213     private:
00214       virtual MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi)=0;
00215   };
00216 
00217   template<class T>
00218     class TreeObjectReader: public TreeReaderBase {
00219       public:
00220         TreeObjectReader():m_tree(0),m_fullName(0),m_buffer(0),m_tag(0){
00221         }
00222         virtual MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) {
00223           m_tree->GetEntry(iIndex);
00224           MonitorElement* element = iStore.get(*m_fullName);
00225           if(0 == element) {
00226             std::string path;
00227             const char* name;
00228             splitName(*m_fullName, path,name);
00229             iStore.setCurrentFolder(path);
00230             element = createElement(iStore,name,m_buffer);
00231             if(iIsLumi) { element->setLumiFlag();}
00232           } else {
00233             mergeWithElement(element,m_buffer);
00234           }
00235           if(0!= m_tag) {
00236             iStore.tag(element,m_tag);
00237           }
00238           return element;
00239         }
00240         virtual void setTree(TTree* iTree)  {
00241           m_tree = iTree;
00242           m_tree->SetBranchAddress(kFullNameBranch,&m_fullName);
00243           m_tree->SetBranchAddress(kFlagBranch,&m_tag);
00244           m_tree->SetBranchAddress(kValueBranch,&m_buffer);
00245         }
00246       private:
00247         TTree* m_tree;
00248         std::string* m_fullName;
00249         T* m_buffer;
00250         uint32_t m_tag;
00251     };
00252 
00253   template<class T>
00254     class TreeSimpleReader : public TreeReaderBase {
00255       public:
00256         TreeSimpleReader():m_tree(0),m_fullName(0),m_buffer(),m_tag(0){
00257         }
00258         virtual MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore,bool iIsLumi) {
00259           m_tree->GetEntry(iIndex);
00260           MonitorElement* element = iStore.get(*m_fullName);
00261           if(0 == element) {
00262             std::string path;
00263             const char* name;
00264             splitName(*m_fullName, path,name);
00265             iStore.setCurrentFolder(path);
00266             element = createElement(iStore,name,m_buffer);
00267             if(iIsLumi) { element->setLumiFlag();}
00268           } else {
00269             mergeWithElement(element, m_buffer);
00270           }
00271           if(0!=m_tag) {
00272             iStore.tag(element,m_tag);
00273           }
00274           return element;
00275         }
00276         virtual void setTree(TTree* iTree)  {
00277           m_tree = iTree;
00278           m_tree->SetBranchAddress(kFullNameBranch,&m_fullName);
00279           m_tree->SetBranchAddress(kFlagBranch,&m_tag);
00280           m_tree->SetBranchAddress(kValueBranch,&m_buffer);
00281         }
00282       private:
00283         TTree* m_tree;
00284         std::string* m_fullName;
00285         T m_buffer;
00286         uint32_t m_tag;
00287     };
00288 
00289 }
00290 
00291 class DQMRootSource : public edm::InputSource
00292 {
00293 
00294    public:
00295       DQMRootSource(edm::ParameterSet const&, const edm::InputSourceDescription&);
00296       virtual ~DQMRootSource();
00297 
00298       // ---------- const member functions ---------------------
00299 
00300       // ---------- static member functions --------------------
00301 
00302       // ---------- member functions ---------------------------
00303       static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
00304 
00305    private:
00306 
00307       DQMRootSource(const DQMRootSource&); // stop default
00308 
00309       class RunPHIDKey {
00310       public:
00311         RunPHIDKey(edm::ProcessHistoryID const& phid, unsigned int run) : 
00312           processHistoryID_(phid), run_(run) { }
00313         edm::ProcessHistoryID const& processHistoryID() const { return processHistoryID_; }
00314         unsigned int run() const { return run_; }
00315         bool operator<(RunPHIDKey const& right) const {
00316           if (processHistoryID_ == right.processHistoryID()) {
00317             return run_ < right.run();
00318           }
00319           return processHistoryID_ < right.processHistoryID();
00320         }
00321       private:
00322         edm::ProcessHistoryID processHistoryID_;
00323         unsigned int run_;
00324       };
00325 
00326       class RunLumiPHIDKey {
00327       public:
00328         RunLumiPHIDKey(edm::ProcessHistoryID const& phid, unsigned int run, unsigned int lumi) : 
00329           processHistoryID_(phid), run_(run), lumi_(lumi) { }
00330         edm::ProcessHistoryID const& processHistoryID() const { return processHistoryID_; }
00331         unsigned int run() const { return run_; }
00332         unsigned int lumi() const { return lumi_; }
00333         bool operator<(RunLumiPHIDKey const& right) const {
00334           if (processHistoryID_ == right.processHistoryID()) {
00335             if (run_ == right.run()) {
00336               return lumi_ < right.lumi();
00337             }
00338             return run_ < right.run();
00339           }
00340           return processHistoryID_ < right.processHistoryID();
00341         }
00342       private:
00343         edm::ProcessHistoryID processHistoryID_;
00344         unsigned int run_;
00345         unsigned int lumi_;
00346       };
00347 
00348       virtual edm::InputSource::ItemType getNextItemType();
00349       //NOTE: the following is really read next run auxiliary
00350       virtual boost::shared_ptr<edm::RunAuxiliary> readRunAuxiliary_() ;
00351       virtual boost::shared_ptr<edm::LuminosityBlockAuxiliary> readLuminosityBlockAuxiliary_() ;
00352       virtual boost::shared_ptr<edm::RunPrincipal> readRun_(boost::shared_ptr<edm::RunPrincipal> rpCache);
00353       virtual boost::shared_ptr<edm::LuminosityBlockPrincipal> readLuminosityBlock_( boost::shared_ptr<edm::LuminosityBlockPrincipal> lbCache);
00354       virtual edm::EventPrincipal* readEvent_(edm::EventPrincipal&) ;
00355       
00356       virtual std::unique_ptr<edm::FileBlock> readFile_();
00357       virtual void closeFile_();
00358       
00359       void logFileAction(char const* msg, char const* fileName) const;
00360       
00361       void readNextItemType();
00362       void setupFile(unsigned int iIndex);
00363       void readElements();
00364       
00365       const DQMRootSource& operator=(const DQMRootSource&); // stop default
00366 
00367       // ---------- member data --------------------------------
00368       edm::InputFileCatalog m_catalog;
00369       edm::RunAuxiliary m_runAux;
00370       edm::LuminosityBlockAuxiliary m_lumiAux;
00371       edm::InputSource::ItemType m_nextItemType;
00372 
00373       size_t m_fileIndex;
00374       size_t m_presentlyOpenFileIndex;
00375       std::list<unsigned int>::iterator m_nextIndexItr;
00376       std::list<unsigned int>::iterator m_presentIndexItr;
00377       std::vector<RunLumiToRange> m_runlumiToRange;
00378       std::auto_ptr<TFile> m_file;
00379       std::vector<TTree*> m_trees;
00380       std::vector<boost::shared_ptr<TreeReaderBase> > m_treeReaders;
00381       
00382       std::list<unsigned int> m_orderedIndices;
00383       edm::ProcessHistoryID m_lastSeenReducedPHID;
00384       unsigned int m_lastSeenRun;
00385       edm::ProcessHistoryID m_lastSeenReducedPHID2;
00386       unsigned int m_lastSeenRun2;
00387       unsigned int m_lastSeenLumi2;
00388       unsigned int m_filterOnRun;
00389       bool m_justOpenedFileSoNeedToGenerateRunTransition;
00390       bool m_shouldReadMEs;
00391       std::set<MonitorElement*> m_lumiElements;
00392       std::set<MonitorElement*> m_runElements;
00393       std::vector<edm::ProcessHistoryID> m_historyIDs;
00394       std::vector<edm::ProcessHistoryID> m_reducedHistoryIDs;
00395       
00396       edm::JobReport::Token m_jrToken;
00397 };
00398 
00399 //
00400 // constants, enums and typedefs
00401 //
00402 
00403 //
00404 // static data member definitions
00405 //
00406 
00407 void
00408 DQMRootSource::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
00409   edm::ParameterSetDescription desc;
00410   desc.addUntracked<std::vector<std::string> >("fileNames")
00411     ->setComment("Names of files to be processed.");
00412   desc.addUntracked<unsigned int>("filterOnRun",0)
00413     ->setComment("Just limit the process to the selected run.");
00414   desc.addUntracked<std::string>("overrideCatalog",std::string())
00415     ->setComment("An alternate file catalog to use instead of the standard site one.");
00416   descriptions.addDefault(desc);
00417 }
00418 //
00419 // constructors and destructor
00420 //
00421 DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSourceDescription& iDesc):
00422   edm::InputSource(iPSet,iDesc),
00423   m_catalog(iPSet.getUntrackedParameter<std::vector<std::string> >("fileNames"),
00424             iPSet.getUntrackedParameter<std::string>("overrideCatalog")),
00425   m_nextItemType(edm::InputSource::IsFile),
00426   m_fileIndex(0),
00427   m_presentlyOpenFileIndex(0),
00428   m_trees(kNIndicies,static_cast<TTree*>(0)),
00429   m_treeReaders(kNIndicies,boost::shared_ptr<TreeReaderBase>()),
00430   m_lastSeenReducedPHID(),
00431   m_lastSeenRun(0),
00432   m_lastSeenReducedPHID2(),
00433   m_lastSeenRun2(0),
00434   m_lastSeenLumi2(0),
00435   m_filterOnRun(iPSet.getUntrackedParameter<unsigned int>("filterOnRun", 0)),
00436   m_justOpenedFileSoNeedToGenerateRunTransition(false),
00437   m_shouldReadMEs(true)
00438 {
00439   if(m_fileIndex ==m_catalog.fileNames().size()) {
00440     m_nextItemType=edm::InputSource::IsStop;
00441   } else{
00442     m_treeReaders[kIntIndex].reset(new TreeSimpleReader<Long64_t>());
00443     m_treeReaders[kFloatIndex].reset(new TreeSimpleReader<double>());
00444     m_treeReaders[kStringIndex].reset(new TreeObjectReader<std::string>());
00445     m_treeReaders[kTH1FIndex].reset(new TreeObjectReader<TH1F>());
00446     m_treeReaders[kTH1SIndex].reset(new TreeObjectReader<TH1S>());
00447     m_treeReaders[kTH1DIndex].reset(new TreeObjectReader<TH1D>());
00448     m_treeReaders[kTH2FIndex].reset(new TreeObjectReader<TH2F>());
00449     m_treeReaders[kTH2SIndex].reset(new TreeObjectReader<TH2S>());
00450     m_treeReaders[kTH2DIndex].reset(new TreeObjectReader<TH2D>());
00451     m_treeReaders[kTH3FIndex].reset(new TreeObjectReader<TH3F>());
00452     m_treeReaders[kTProfileIndex].reset(new TreeObjectReader<TProfile>());
00453     m_treeReaders[kTProfile2DIndex].reset(new TreeObjectReader<TProfile2D>());
00454   }
00455 
00456 }
00457 
00458 // DQMRootSource::DQMRootSource(const DQMRootSource& rhs)
00459 // {
00460 //    // do actual copying here;
00461 // }
00462 
00463 DQMRootSource::~DQMRootSource()
00464 {
00465   if(m_file.get() != 0 && m_file->IsOpen()) {
00466     m_file->Close();
00467     logFileAction("  Closed file ", m_catalog.fileNames()[m_presentlyOpenFileIndex].c_str());
00468   }
00469 }
00470 
00471 //
00472 // assignment operators
00473 //
00474 // const DQMRootSource& DQMRootSource::operator=(const DQMRootSource& rhs)
00475 // {
00476 //   //An exception safe implementation is
00477 //   DQMRootSource temp(rhs);
00478 //   swap(rhs);
00479 //
00480 //   return *this;
00481 // }
00482 
00483 //
00484 // member functions
00485 //
00486 edm::EventPrincipal* DQMRootSource::readEvent_(edm::EventPrincipal&)
00487 {
00488   return 0;
00489 }
00490 
00491 edm::InputSource::ItemType DQMRootSource::getNextItemType()
00492 {
00493   //std::cout <<"getNextItemType "<<m_nextItemType<<std::endl;
00494   return m_nextItemType;
00495 }
00496 
00497 boost::shared_ptr<edm::RunAuxiliary> DQMRootSource::readRunAuxiliary_()
00498 {
00499   //std::cout <<"readRunAuxiliary_"<<std::endl;
00500   assert(m_nextIndexItr != m_orderedIndices.end());
00501   RunLumiToRange runLumiRange = m_runlumiToRange[*m_nextIndexItr];
00502 
00503   //NOTE: the setBeginTime and setEndTime functions of RunAuxiliary only work if the previous value was invalid
00504   // therefore we must copy
00505   m_runAux = edm::RunAuxiliary(runLumiRange.m_run,edm::Timestamp(runLumiRange.m_beginTime),edm::Timestamp(runLumiRange.m_endTime));
00506   assert(m_historyIDs.size() > runLumiRange.m_historyIDIndex);
00507   //std::cout <<"readRunAuxiliary_ "<<m_historyIDs[runLumiRange.m_historyIDIndex]<<std::endl;
00508   m_runAux.setProcessHistoryID(m_historyIDs[runLumiRange.m_historyIDIndex]);    
00509   return boost::shared_ptr<edm::RunAuxiliary>( new edm::RunAuxiliary(m_runAux) );
00510 }
00511 
00512 boost::shared_ptr<edm::LuminosityBlockAuxiliary>
00513 DQMRootSource::readLuminosityBlockAuxiliary_()
00514 {
00515   //std::cout <<"readLuminosityBlockAuxiliary_"<<std::endl;
00516   assert(m_nextIndexItr != m_orderedIndices.end());
00517   const RunLumiToRange runLumiRange = m_runlumiToRange[*m_nextIndexItr];
00518   m_lumiAux = edm::LuminosityBlockAuxiliary(edm::LuminosityBlockID(runLumiRange.m_run,runLumiRange.m_lumi),
00519                                             edm::Timestamp(runLumiRange.m_beginTime),
00520                                             edm::Timestamp(runLumiRange.m_endTime));
00521   assert(m_historyIDs.size() > runLumiRange.m_historyIDIndex);
00522   //std::cout <<"lumi "<<m_lumiAux.beginTime().value()<<" "<<runLumiRange.m_beginTime<<std::endl;
00523   m_lumiAux.setProcessHistoryID(m_historyIDs[runLumiRange.m_historyIDIndex]);    
00524   
00525   return boost::shared_ptr<edm::LuminosityBlockAuxiliary>(new edm::LuminosityBlockAuxiliary(m_lumiAux));
00526 }
00527 
00528 boost::shared_ptr<edm::RunPrincipal>
00529 DQMRootSource::readRun_(boost::shared_ptr<edm::RunPrincipal> rpCache)
00530 {
00531   assert(m_presentIndexItr != m_orderedIndices.end());
00532   RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr];
00533 
00534   m_justOpenedFileSoNeedToGenerateRunTransition = false;
00535   unsigned int runID =rpCache->id().run();
00536   assert(runID == runLumiRange.m_run);
00537 
00538   m_shouldReadMEs = (m_filterOnRun == 0 ||
00539                      (m_filterOnRun != 0 && m_filterOnRun == runID)); 
00540   //   std::cout <<"readRun_"<<std::endl;
00541   //   std::cout <<"m_shouldReadMEs " << m_shouldReadMEs <<std::endl;
00542 
00553   //NOTE: need to reset all run elements at this point
00554   if( m_lastSeenRun != runID ||
00555       m_lastSeenReducedPHID != m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex) ) {
00556     if (m_shouldReadMEs) {
00557       edm::Service<DQMStore> store;
00558       std::vector<MonitorElement*> allMEs = (*store).getAllContents("");
00559       for(auto const& ME : allMEs) {
00560         // We do not want to reset here Lumi products, since a dedicated
00561         // resetting is done at every lumi transition.
00562         if (ME->getLumiFlag()) continue;
00563         if ( !(*store).isCollate() )
00564           ME->Reset();
00565       }
00566     }
00567     m_lastSeenReducedPHID = m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex);
00568     m_lastSeenRun = runID;
00569   }
00570 
00571   readNextItemType();
00572 
00573   //NOTE: it is possible to have a Run when all we have stored is lumis
00574   if(runLumiRange.m_lumi == 0) {
00575     readElements();
00576   }
00577 
00578   edm::Service<edm::JobReport> jr;
00579   jr->reportInputRunNumber(rpCache->id().run());
00580 
00581   rpCache->fillRunPrincipal();
00582   return rpCache;
00583 }
00584 
00585 boost::shared_ptr<edm::LuminosityBlockPrincipal>
00586 DQMRootSource::readLuminosityBlock_( boost::shared_ptr<edm::LuminosityBlockPrincipal> lbCache)
00587 {
00588   assert(m_presentIndexItr != m_orderedIndices.end());
00589   RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr];
00590   assert(runLumiRange.m_run == lbCache->id().run());
00591   assert(runLumiRange.m_lumi == lbCache->id().luminosityBlock());
00592 
00593   //NOTE: need to reset all lumi block elements at this point
00594   if( ( m_lastSeenLumi2 != runLumiRange.m_lumi ||
00595         m_lastSeenRun2 != runLumiRange.m_run ||
00596         m_lastSeenReducedPHID2 != m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex) )
00597       && m_shouldReadMEs) {
00598 
00599     edm::Service<DQMStore> store;
00600     std::vector<MonitorElement*> allMEs = (*store).getAllContents("");
00601     for(auto const& ME : allMEs) {
00602       // We do not want to reset Run Products here!
00603       if (ME->getLumiFlag()) {
00604         ME->Reset();
00605       }
00606     }
00607     m_lastSeenReducedPHID2 = m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex);
00608     m_lastSeenRun2 = runLumiRange.m_run;
00609     m_lastSeenLumi2 = runLumiRange.m_lumi;
00610   }
00611   readNextItemType();
00612   //std::cout <<"readLuminosityBlock_"<<std::endl;
00613 
00614   readElements();
00615 
00616   edm::Service<edm::JobReport> jr;
00617   jr->reportInputLumiSection(lbCache->id().run(),lbCache->id().luminosityBlock());
00618 
00619   lbCache->fillLuminosityBlockPrincipal();
00620   return lbCache;
00621 }
00622 
00623 std::unique_ptr<edm::FileBlock>
00624 DQMRootSource::readFile_() {
00625   //std::cout <<"readFile_"<<std::endl;
00626   setupFile(m_fileIndex);
00627   ++m_fileIndex;
00628   readNextItemType();
00629 
00630   edm::Service<edm::JobReport> jr;
00631   m_jrToken = jr->inputFileOpened(m_catalog.fileNames()[m_fileIndex-1],
00632       m_catalog.logicalFileNames()[m_fileIndex-1],
00633       std::string(),
00634       std::string(),
00635       "DQMRootSource",
00636       "source",
00637       m_file->GetUUID().AsString(),//edm::createGlobalIdentifier(),
00638       std::vector<std::string>()
00639       );
00640 
00641   return std::unique_ptr<edm::FileBlock>(new edm::FileBlock);
00642 }
00643 
00644 void
00645 DQMRootSource::closeFile_() {
00646   edm::Service<edm::JobReport> jr;
00647   jr->inputFileClosed(m_jrToken);
00648 }
00649 
00650 void DQMRootSource::readElements() {
00651   edm::Service<DQMStore> store;
00652   RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr];
00653   bool shouldContinue = false;
00654   do
00655   {
00656     shouldContinue = false;
00657     ++m_presentIndexItr;
00658     if(runLumiRange.m_type == kNoTypesStored) {continue;}
00659     boost::shared_ptr<TreeReaderBase> reader = m_treeReaders[runLumiRange.m_type];
00660     ULong64_t index = runLumiRange.m_firstIndex;
00661     ULong64_t endIndex = runLumiRange.m_lastIndex+1;
00662     for (; index != endIndex; ++index)
00663     {
00664       bool isLumi = runLumiRange.m_lumi !=0;
00665       if (m_shouldReadMEs)
00666         reader->read(index,*store,isLumi);
00667 //       if (isLumi)
00668 //       {
00669 //         std::cout << runLumiRange.m_run << " " << runLumiRange.m_lumi << "  lumi element "<< element->getFullname()<<" "<<index<< " " << runLumiRange.m_type << std::endl;
00670 //         m_lumiElements.insert(element);
00671 //       }
00672 //       else
00673 //       {
00674 //         std::cout << runLumiRange.m_run << " " << runLumiRange.m_lumi << "  run element "<< element->getFullname()<<" "<<index<< " " << runLumiRange.m_type << std::endl;
00675 //         m_runElements.insert(element);
00676 //       }
00677     }
00678     if (m_presentIndexItr != m_orderedIndices.end())
00679     {
00680       //are there more parts to this same run/lumi?
00681       const RunLumiToRange nextRunLumiRange = m_runlumiToRange[*m_presentIndexItr];
00682       //continue to the next item if that item is either
00683       if ( (m_reducedHistoryIDs.at(nextRunLumiRange.m_historyIDIndex) == m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex)) &&
00684           (nextRunLumiRange.m_run == runLumiRange.m_run) &&
00685           (nextRunLumiRange.m_lumi == runLumiRange.m_lumi) )
00686       {
00687         shouldContinue= true;
00688         runLumiRange = nextRunLumiRange;
00689       }
00690     }
00691   } while(shouldContinue);
00692 }
00693 
00694 void DQMRootSource::readNextItemType()
00695 {
00696   //Do the work of actually figuring out where next to go
00697 
00698   assert (m_nextIndexItr != m_orderedIndices.end());
00699   RunLumiToRange runLumiRange = m_runlumiToRange[*m_nextIndexItr];
00700 
00701   if (m_nextItemType != edm::InputSource::IsFile) {
00702     if (runLumiRange.m_lumi != 0 && m_nextItemType == edm::InputSource::IsRun) {
00703       m_nextItemType = edm::InputSource::IsLumi;
00704       return;
00705     }
00706     ++m_nextIndexItr;
00707   }
00708   else
00709   {
00710     //NOTE: the following makes the iterator not be advanced in the
00711     //do while loop below.
00712     runLumiRange.m_run=0;
00713   }
00714 
00715   bool shouldContinue = false;
00716   do
00717   {
00718     shouldContinue = false;
00719     if (m_nextIndexItr == m_orderedIndices.end())
00720     {
00721       //go to next file
00722       m_nextItemType = edm::InputSource::IsFile;
00723       //std::cout <<"going to next file"<<std::endl;
00724       if(m_fileIndex == m_catalog.fileNames().size()) {
00725         m_nextItemType = edm::InputSource::IsStop;
00726       }       
00727       break;
00728     }
00729     const RunLumiToRange nextRunLumiRange = m_runlumiToRange[*m_nextIndexItr];
00730     //continue to the next item if that item is the same run or lumi as we just did
00731     if(  (m_reducedHistoryIDs.at(nextRunLumiRange.m_historyIDIndex) == m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex) ) &&
00732          (nextRunLumiRange.m_run == runLumiRange.m_run) &&
00733          (nextRunLumiRange.m_lumi == runLumiRange.m_lumi) ) {
00734       shouldContinue= true;
00735       ++m_nextIndexItr;
00736       //std::cout <<"  advancing " <<nextRunLumiRange.m_run<<" "<<nextRunLumiRange.m_lumi<<std::endl;
00737     } 
00738   } while(shouldContinue);
00739   
00740   if(m_nextIndexItr != m_orderedIndices.end()) {
00741     if (m_justOpenedFileSoNeedToGenerateRunTransition ||
00742         m_lastSeenRun != m_runlumiToRange[*m_nextIndexItr].m_run ||
00743         m_lastSeenReducedPHID != m_reducedHistoryIDs.at(m_runlumiToRange[*m_nextIndexItr].m_historyIDIndex) ) {
00744       m_nextItemType = edm::InputSource::IsRun;
00745     } else {
00746         m_nextItemType = edm::InputSource::IsLumi;
00747     }
00748   }
00749 }
00750 
00751 void 
00752 DQMRootSource::setupFile(unsigned int iIndex)
00753 {
00754   if(m_file.get() != 0 && iIndex > 0) {
00755     m_file->Close();
00756     logFileAction("  Closed file ", m_catalog.fileNames()[iIndex-1].c_str());
00757   }
00758   logFileAction("  Initiating request to open file ", m_catalog.fileNames()[iIndex].c_str());
00759   m_presentlyOpenFileIndex = iIndex;
00760   try {
00761     m_file = std::auto_ptr<TFile>(TFile::Open(m_catalog.fileNames()[iIndex].c_str()));
00762   } catch(cms::Exception const& e) {
00763     edm::Exception ex(edm::errors::FileOpenError,"",e);
00764     ex.addContext("Opening DQM Root file");
00765     ex <<"\nInput file " << m_catalog.fileNames()[iIndex] << " was not found, could not be opened, or is corrupted.\n";
00766     throw ex;
00767   }
00768   if(not m_file->IsZombie()) {  
00769     logFileAction("  Successfully opened file ", m_catalog.fileNames()[iIndex].c_str());
00770   } else {
00771     edm::Exception ex(edm::errors::FileOpenError);
00772     ex<<"Input file "<<m_catalog.fileNames()[iIndex].c_str() <<" could not be opened.\n";
00773     ex.addContext("Opening DQM Root file");
00774     throw ex;
00775   }
00776   //Check file format version, which is encoded in the Title of the TFile
00777   if(0 != strcmp(m_file->GetTitle(),"1")) {
00778     edm::Exception ex(edm::errors::FileReadError);
00779     ex<<"Input file "<<m_catalog.fileNames()[iIndex].c_str() <<" does not appear to be a DQM Root file.\n";
00780   }
00781   
00782   //Get meta Data
00783   TDirectory* metaDir = m_file->GetDirectory(kMetaDataDirectoryAbsolute);
00784   if(0==metaDir) {
00785     edm::Exception ex(edm::errors::FileReadError);
00786     ex<<"Input file "<<m_catalog.fileNames()[iIndex].c_str() <<" appears to be corrupted since it does not have the proper internal structure.\n"
00787       " Check to see if the file was closed properly.\n";    
00788     ex.addContext("Opening DQM Root file");
00789     throw ex;    
00790   }
00791   TTree* parameterSetTree = dynamic_cast<TTree*>(metaDir->Get(kParameterSetTree));
00792   assert(0!=parameterSetTree);
00793 
00794   edm::pset::Registry* psr = edm::pset::Registry::instance();
00795   assert(0!=psr);
00796   {
00797     std::string blob;
00798     std::string* pBlob = &blob;
00799     parameterSetTree->SetBranchAddress(kParameterSetBranch,&pBlob);
00800     for(unsigned int index = 0; index != parameterSetTree->GetEntries();++index)
00801     {
00802       parameterSetTree->GetEntry(index);
00803       cms::Digest dg(blob);
00804       edm::ParameterSetID psID(dg.digest().toString());
00805       edm::ParameterSet temp(blob,psID);
00806     } 
00807   }
00808 
00809   {
00810     TTree* processHistoryTree = dynamic_cast<TTree*>(metaDir->Get(kProcessHistoryTree));
00811     assert(0!=processHistoryTree);
00812     unsigned int phIndex = 0;
00813     processHistoryTree->SetBranchAddress(kPHIndexBranch,&phIndex);
00814     std::string processName;
00815     std::string* pProcessName = &processName;
00816     processHistoryTree->SetBranchAddress(kProcessConfigurationProcessNameBranch,&pProcessName);
00817     std::string parameterSetIDBlob;
00818     std::string* pParameterSetIDBlob = &parameterSetIDBlob;
00819     processHistoryTree->SetBranchAddress(kProcessConfigurationParameterSetIDBranch,&pParameterSetIDBlob);
00820     std::string releaseVersion;
00821     std::string* pReleaseVersion = &releaseVersion;
00822     processHistoryTree->SetBranchAddress(kProcessConfigurationReleaseVersion,&pReleaseVersion);
00823     std::string passID;
00824     std::string* pPassID = &passID;
00825     processHistoryTree->SetBranchAddress(kProcessConfigurationPassID,&pPassID);
00826 
00827     edm::ProcessConfigurationRegistry* pcr = edm::ProcessConfigurationRegistry::instance();
00828     assert(0!=pcr);
00829     edm::ProcessHistoryRegistry* phr = edm::ProcessHistoryRegistry::instance();
00830     assert(0!=phr);
00831     std::vector<edm::ProcessConfiguration> configs;
00832     configs.reserve(5);
00833     m_historyIDs.clear();
00834     m_reducedHistoryIDs.clear();
00835     for(unsigned int i=0; i != processHistoryTree->GetEntries(); ++i) {
00836       processHistoryTree->GetEntry(i);
00837       if(phIndex==0) {
00838         if(not configs.empty()) {
00839           edm::ProcessHistory ph(configs);
00840           m_historyIDs.push_back(ph.id());
00841           phr->insertMapped(ph);
00842           m_reducedHistoryIDs.push_back(phr->extra().reduceProcessHistoryID(ph.id()));
00843         }
00844         configs.clear();
00845       }
00846       edm::ParameterSetID psetID(parameterSetIDBlob);
00847       edm::ProcessConfiguration pc(processName, psetID,releaseVersion,passID);
00848       pcr->insertMapped(pc);
00849       configs.push_back(pc);
00850     }
00851     if(not configs.empty()) {
00852       edm::ProcessHistory ph(configs);
00853       m_historyIDs.push_back(ph.id());
00854       phr->insertMapped( ph);
00855       m_reducedHistoryIDs.push_back(phr->extra().reduceProcessHistoryID(ph.id()));
00856       //std::cout <<"inserted "<<ph.id()<<std::endl;
00857     }
00858   }
00859 
00860   //Setup the indices
00861   TTree* indicesTree = dynamic_cast<TTree*>(m_file->Get(kIndicesTree));
00862   assert(0!=indicesTree);
00863 
00864   m_runlumiToRange.clear();
00865   m_runlumiToRange.reserve(indicesTree->GetEntries());
00866   m_orderedIndices.clear();
00867 
00868   RunLumiToRange temp;
00869   indicesTree->SetBranchAddress(kRunBranch,&temp.m_run);
00870   indicesTree->SetBranchAddress(kLumiBranch,&temp.m_lumi);
00871   indicesTree->SetBranchAddress(kBeginTimeBranch,&temp.m_beginTime);
00872   indicesTree->SetBranchAddress(kEndTimeBranch,&temp.m_endTime);
00873   indicesTree->SetBranchAddress(kProcessHistoryIndexBranch,&temp.m_historyIDIndex);
00874   indicesTree->SetBranchAddress(kTypeBranch,&temp.m_type);
00875   indicesTree->SetBranchAddress(kFirstIndex,&temp.m_firstIndex);
00876   indicesTree->SetBranchAddress(kLastIndex,&temp.m_lastIndex);
00877 
00878   //Need to reorder items since if there was a merge done the same Run
00879   //and/or Lumi can appear multiple times but we want to process them
00880   //all at once
00881 
00882   //We use a std::list for m_orderedIndices since inserting into the
00883   //middle of a std::list does not disrupt the iterators to already
00884   //existing entries
00885 
00886   //The Map is used to see if a Run/Lumi pair has appeared before
00887   typedef std::map<RunLumiPHIDKey, std::list<unsigned int>::iterator > RunLumiToLastEntryMap;
00888   RunLumiToLastEntryMap runLumiToLastEntryMap;
00889 
00890   //Need to group all lumis for the same run together and move the run
00891   //entry to the beginning
00892   typedef std::map<RunPHIDKey, std::pair< std::list<unsigned int>::iterator, std::list<unsigned int>::iterator> > RunToFirstLastEntryMap;
00893   RunToFirstLastEntryMap runToFirstLastEntryMap;
00894 
00895   for (Long64_t index = 0; index != indicesTree->GetEntries(); ++index)
00896   {
00897     indicesTree->GetEntry(index);
00898 //     std::cout <<"read r:"<<temp.m_run
00899 //            <<" l:"<<temp.m_lumi
00900 //            <<" b:"<<temp.m_beginTime
00901 //            <<" e:"<<temp.m_endTime
00902 //            <<" fi:" << temp.m_firstIndex
00903 //            <<" li:" << temp.m_lastIndex
00904 //            <<" type:" << temp.m_type << std::endl;
00905     m_runlumiToRange.push_back(temp);
00906 
00907     RunLumiPHIDKey runLumi(m_reducedHistoryIDs.at(temp.m_historyIDIndex), temp.m_run, temp.m_lumi);
00908     RunPHIDKey runKey(m_reducedHistoryIDs.at(temp.m_historyIDIndex), temp.m_run);
00909 
00910     RunLumiToLastEntryMap::iterator itFind = runLumiToLastEntryMap.find(runLumi);
00911     if (itFind == runLumiToLastEntryMap.end())
00912     {
00913       //does not already exist
00914       //does the run for this already exist?
00915       std::list<unsigned int>::iterator itLastOfRun = m_orderedIndices.end();
00916 
00917       RunToFirstLastEntryMap::iterator itRunFirstLastEntryFind = runToFirstLastEntryMap.find(runKey);
00918       bool needNewEntryInRunFirstLastEntryMap = true;
00919       if (itRunFirstLastEntryFind != runToFirstLastEntryMap.end())
00920       {
00921         needNewEntryInRunFirstLastEntryMap=false;
00922         if (temp.m_lumi!=0)
00923         {
00924           //lumis go to the end
00925           itLastOfRun = itRunFirstLastEntryFind->second.second;
00926           //we want to insert after this one so must advance the iterator
00927           ++itLastOfRun;
00928         }
00929         else
00930         {
00931           //runs go at the beginning
00932           itLastOfRun = itRunFirstLastEntryFind->second.first;
00933         }
00934       }
00935       std::list<unsigned int>::iterator iter = m_orderedIndices.insert(itLastOfRun,index);
00936       runLumiToLastEntryMap[runLumi]=iter;
00937       if (needNewEntryInRunFirstLastEntryMap)
00938         runToFirstLastEntryMap[runKey]=std::make_pair(iter,iter);
00939       else
00940       {
00941         if(temp.m_lumi!=0)
00942         {
00943           //lumis go at end
00944           runToFirstLastEntryMap[runKey].second = iter;
00945         }
00946         else
00947         {
00948           //since we haven't yet seen this run/lumi combination it means we haven't yet seen
00949           // a run so we can put this first
00950           runToFirstLastEntryMap[runKey].first = iter;
00951         }
00952       }
00953     }
00954     else
00955     {
00956       //We need to do a merge since the run/lumi already appeared. Put it after the existing entry
00957       //std::cout <<" found a second instance of "<<runLumi.first<<" "<<runLumi.second<<" at "<<index<<std::endl;
00958       std::list<unsigned int>::iterator itNext = itFind->second;
00959       ++itNext;
00960       std::list<unsigned int>::iterator iter = m_orderedIndices.insert(itNext,index);
00961       RunToFirstLastEntryMap::iterator itRunFirstLastEntryFind = runToFirstLastEntryMap.find(runKey);
00962       if (itRunFirstLastEntryFind->second.second == itFind->second)
00963       {
00964         //if the previous one was the last in the run, we need to update to make this one the last
00965         itRunFirstLastEntryFind->second.second = iter;
00966       }
00967       itFind->second = iter;
00968     }
00969   }
00970   m_nextIndexItr = m_orderedIndices.begin();
00971   m_presentIndexItr = m_orderedIndices.begin();
00972   
00973   if(m_nextIndexItr != m_orderedIndices.end()) {
00974     for( size_t index = 0; index < kNIndicies; ++index) {
00975       m_trees[index] = dynamic_cast<TTree*>(m_file->Get(kTypeNames[index]));
00976       assert(0!=m_trees[index]);
00977       m_treeReaders[index]->setTree(m_trees[index]);
00978     }
00979   }
00980   //After a file open, the framework expects to see a new 'IsRun'
00981   m_justOpenedFileSoNeedToGenerateRunTransition=true;
00982 }
00983 
00984 void
00985 DQMRootSource::logFileAction(char const* msg, char const* fileName) const {
00986   edm::LogAbsolute("fileAction") << std::setprecision(0) << edm::TimeOfDay() << msg << fileName;
00987   edm::FlushMessageLog();
00988 }
00989 
00990 //
00991 // const member functions
00992 //
00993 
00994 //
00995 // static member functions
00996 //
00997 DEFINE_FWK_INPUT_SOURCE(DQMRootSource);