DQMFileIterator.h

Go to the documentation of this file.

#ifndef IOPool_DQMStreamer_DQMFilerIterator_h
#define IOPool_DQMStreamer_DQMFilerIterator_h

#include "FWCore/ServiceRegistry/interface/Service.h"
#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"

#include "boost/filesystem.hpp"

#include <map>
#include <unordered_set>
#include <chrono>

#include "DQMMonitoringService.h"

namespace dqmservices {

class DQMFileIterator {
 public:
  struct LumiEntry {
    std::string filename;
    std::string run_path;

    unsigned int file_ls;
    std::size_t n_events_processed;
    std::size_t n_events_accepted;
    std::string datafn;

    static LumiEntry load_json(const std::string& run_path, const std::string& filename, int lumiNumber,
                               int datafn_position);

    std::string get_data_path() const;
    std::string get_json_path() const;
    std::string state;
  };

  struct EorEntry {
    bool loaded = false;
    std::string filename;
    std::string run_path;

    std::size_t n_events;
    std::size_t n_lumi;

    static EorEntry load_json(const std::string& run_path, const std::string& filename);
  };

  enum State {
    OPEN = 0,
    EOR_CLOSING = 1,  // EoR file found, but lumis are still pending
    EOR = 2,
  };

  DQMFileIterator(edm::ParameterSet const& pset);
  ~DQMFileIterator();
  void initialise(int run, const std::string&, const std::string&);

  State state();

  /* methods to iterate the actual files */

  /* nextLumiNumber_ is the first unprocessed lumi number
   * lumiReady() returns if the next lumi is ready to be loaded
   * open() opens a file and advances the pointer to the next lumi
   *
   * front() a reference to the description (LumiEntry)
   * pop() advances to the next lumi
   */
  bool lumiReady();
  LumiEntry open();

  void pop();

  /* control */
  void reset();
  void update_state();

  /* misc helpers for input sources */
  void logFileAction(const std::string& msg,
                     const std::string& fileName = "") const;
  void logLumiState(const LumiEntry& lumi, const std::string& msg);

  void delay();

  unsigned int runNumber();
  unsigned int lastLumiFound();
  void advanceToLumi(unsigned int lumi, std::string reason);

  static void fillDescription(edm::ParameterSetDescription& d);

 private:
  unsigned int runNumber_;
  std::string runInputDir_;
  std::string streamLabel_;
  unsigned long delayMillis_;
  long nextLumiTimeoutMillis_;
  long forceFileCheckTimeoutMillis_;
  bool flagScanOnce_;

  // file name position in the json file
  unsigned int datafnPosition_;
  std::vector<std::string> runPath_;

  EorEntry eor_;
  State state_;

  unsigned int nextLumiNumber_;
  std::map<unsigned int, LumiEntry> lumiSeen_;
  std::unordered_set<std::string> filesSeen_;

  /* this should be different,
   * since time between hosts might be not in sync */
  std::time_t runPathMTime_;
  std::chrono::high_resolution_clock::time_point runPathLastCollect_;

  /* this is for missing lumi files */
  std::chrono::high_resolution_clock::time_point lastLumiLoad_;

  std::time_t mtimeHash() const;
  void collect(bool ignoreTimers);
  void monUpdateLumi(const LumiEntry& lumi);

  /* this is for monitoring */
  edm::Service<DQMMonitoringService> mon_;
};

} /* end of namespace */

#endif