DQMStore.cc

Go to the documentation of this file.

#include "DQMServices/Core/interface/Standalone.h"
#include "DQMServices/Core/interface/DQMStore.h"
#include "DQMServices/Core/interface/QReport.h"
#include "DQMServices/Core/interface/QTest.h"
#include "DQMServices/Core/src/ROOTFilePB.pb.h"
#include "DQMServices/Core/src/DQMError.h"
#include "classlib/utils/RegexpMatch.h"
#include "classlib/utils/Regexp.h"
#include "classlib/utils/StringOps.h"
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/gzip_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include "TFile.h"
#include "TROOT.h"
#include "TKey.h"
#include "TClass.h"
#include "TSystem.h"
#include "TBufferFile.h"
#include <iterator>
#include <cerrno>
#include <boost/algorithm/string.hpp>
#include <boost/range/iterator_range_core.hpp>

#include <fstream>
#include <sstream>
#include <exception>
#include <utility>

static const std::string s_monitorDirName = "DQMData";
static const std::string s_referenceDirName = "Reference";
static const std::string s_collateDirName = "Collate";
static const std::string s_safe = "/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-+=_()# ";

static const lat::Regexp s_rxmeval ("^<(.*)>(i|f|s|e|t|qr)=(.*)</\\1>$");
static const lat::Regexp s_rxmeqr1 ("^st:(\\d+):([-+e.\\d]+):([^:]*):(.*)$");
static const lat::Regexp s_rxmeqr2 ("^st\\.(\\d+)\\.(.*)$");
static const lat::Regexp s_rxtrace ("(.*)\\((.*)\\+0x.*\\).*");
static const lat::Regexp s_rxself  ("^[^()]*DQMStore::.*");
static const lat::Regexp s_rxpbfile (".*\\.pb$");

static bool
isSubdirectory(const std::string &ofdir, const std::string &path)
{
  return (ofdir.empty()
          || (path.size() >= ofdir.size()
              && path.compare(0, ofdir.size(), ofdir) == 0
              && (path.size() == ofdir.size()
                  || path[ofdir.size()] == '/')));
}

static void
cleanTrailingSlashes(const std::string &path, std::string &clean, const std::string *&cleaned)
{
  clean.clear();
  cleaned = &path;

  size_t len = path.size();
  for ( ; len > 0 && path[len-1] == '/'; --len)
    ;

  if (len != path.size())
  {
    clean = path.substr(0, len);
    cleaned = &clean;
  }
}

static void
splitPath(std::string &dir, std::string &name, const std::string &path)
{
  size_t slash = path.rfind('/');
  if (slash != std::string::npos)
  {
    dir.append(path, 0, slash);
    name.append(path, slash+1, std::string::npos);
  }
  else
    name = path;
}

static void
mergePath(std::string &path, const std::string &dir, const std::string &name)
{
  path.reserve(dir.size() + name.size() + 2);
  path += dir;
  if (! path.empty())
    path += '/';
  path += name;
}

template <class T>
QCriterion *
makeQCriterion(const std::string &qtname)
{ return new T(qtname); }

template <class T>
void
initQCriterion(std::map<std::string, QCriterion *(*)(const std::string &)> &m)
{ m[T::getAlgoName()] = &makeQCriterion<T>; }


fastmatch::fastmatch (std::string  _fastString) :
  fastString_ (std::move(_fastString)),  matching_ (UseFull)
{
  try
  {
    regexp_ = nullptr;
    regexp_ = new lat::Regexp(fastString_, 0, lat::Regexp::Wildcard);
    regexp_->study();
  }
  catch (lat::Error &e)
  {
    delete regexp_;
    raiseDQMError("DQMStore", "Invalid wildcard pattern '%s' in quality"
                  " test specification", fastString_.c_str());
  }

  // count stars ( "*" )
  size_t starCount = 0;
  int pos = -1;
  while (true)
  {
    pos = fastString_.find('*', pos + 1 );
    if ((size_t)pos == std::string::npos)
      break;
    starCount ++;
  }

  // investigate for heuristics
  if ((fastString_.find('"') != std::string::npos)  ||
      (fastString_.find(']') != std::string::npos)  ||
      (fastString_.find('?') != std::string::npos)  ||
      (fastString_.find('\\') != std::string::npos) ||
      (starCount > 2))
  {
    // no fast version can be used
    return;
  }

  // match for pattern "*MyString" and "MyString*"
  if (starCount == 1)
  {
    if (boost::algorithm::starts_with(fastString_, "*"))
    {
      matching_ = OneStarStart;
      fastString_.erase(0,1);
      return;
    }

    if (boost::algorithm::ends_with(fastString_, "*"))
    {
      matching_ = OneStarEnd;
      fastString_.erase(fastString_.length()-1,1);
      return;
    }
  }

  // match for pattern "*MyString*"
  if (starCount == 2)
  {
    if (boost::algorithm::starts_with(fastString_, "*") &&
        boost::algorithm::ends_with(fastString_, "*"))
    {
      matching_ = TwoStar;
      fastString_.erase(0,1);
      fastString_.erase(fastString_.size() - 1, 1);
      return;
    }
  }
}

fastmatch::~fastmatch()
{
  if (regexp_ != nullptr)
    delete regexp_;
}

bool fastmatch::compare_strings_reverse(std::string const& pattern,
                                        std::string const& input) const
{
  if (input.size() < pattern.size())
    return false;

  // compare the two strings character by character for equalness:
  // this does not create uneeded copies of std::string. The
  // boost::algorithm implementation does
  std::string::const_reverse_iterator rit_pattern = pattern.rbegin();
  std::string::const_reverse_iterator rit_input = input.rbegin();

  for (; rit_pattern < pattern.rend(); rit_pattern++, rit_input++)
  {
    if (*rit_pattern != *rit_input)
      // found a difference, fail
      return false;
  }
  return true;
}

bool fastmatch::compare_strings(std::string const& pattern,
                                std::string const& input) const
{
  if (input.size() < pattern.size())
    return false;

  // compare the two strings character by character for equalness:
  // this does not create uneeded copies of std::string. The
  // boost::algorithm implementation does.
  std::string::const_iterator rit_pattern = pattern.begin();
  std::string::const_iterator rit_input = input.begin();

  for (; rit_pattern < pattern.end(); rit_pattern++, rit_input++)
  {
    if (*rit_pattern != *rit_input)
      // found a difference, fail
      return false;
  }
  return true;
}

bool fastmatch::match(std::string const& s) const
{
  switch (matching_)
  {
  case OneStarStart:
    return compare_strings_reverse(fastString_, s);

  case OneStarEnd:
    return compare_strings(fastString_, s);

  case TwoStar:
    return (s.find(fastString_) != std::string::npos);

  default:
    return regexp_->match(s);
  }
}

//IBooker methods
void DQMStore::IBooker::cd() {
  owner_->cd();
}

void DQMStore::IBooker::cd(const std::string &dir) {
  owner_->cd(dir);
}

void DQMStore::IBooker::setCurrentFolder(const std::string &fullpath) {
  owner_->setCurrentFolder(fullpath);
}

void DQMStore::IBooker::goUp() {
  owner_->goUp();
}

const std::string & DQMStore::IBooker::pwd() {
  return owner_->pwd();
}

void DQMStore::IBooker::tag(MonitorElement *me, unsigned int tag) {
  owner_->tag(me, tag);
}

void DQMStore::IBooker::tagContents(const std::string &path, unsigned int myTag) {
  owner_->tagContents(path, myTag);
}

//IGetter methods
std::vector<MonitorElement*>
DQMStore::IGetter::getAllContents(const std::string &path,
                  uint32_t runNumber /* = 0 */,
                  uint32_t lumi      /* = 0 */) {
  return owner_->getAllContents(path, runNumber, lumi);
}

MonitorElement * DQMStore::IGetter::get(const std::string &path) {
  return owner_->get(path);
}

MonitorElement * DQMStore::IGetter::getElement(const std::string &path) {
    MonitorElement *ptr = this->get(path);
    if (ptr == nullptr) {
      std::stringstream msg;
      msg << "DQM object not found";

      msg << ": " << path;

      // can't use cms::Exception inside DQMStore
      throw std::out_of_range(msg.str());
    }
    return ptr;
}

std::vector<std::string> DQMStore::IGetter::getSubdirs() {
  return owner_->getSubdirs();
}

std::vector<std::string> DQMStore::IGetter::getMEs() {
  return owner_->getMEs();
}

bool DQMStore::IGetter::containsAnyMonitorable(const std::string &path) {
  return owner_->containsAnyMonitorable(path);
}

bool DQMStore::IGetter::dirExists(const std::string &path) {
  return owner_->dirExists(path);
}

void DQMStore::IGetter::cd() {
  owner_->cd();
}

void DQMStore::IGetter::cd(const std::string &dir) {
  owner_->cd(dir);
}

void DQMStore::IGetter::setCurrentFolder(const std::string &fullpath) {
  owner_->setCurrentFolder(fullpath);
}

void DQMStore::mergeAndResetMEsRunSummaryCache(uint32_t run,
                                               uint32_t streamId,
                                               uint32_t moduleId) {
  if (verbose_ > 1)
    std::cout << "DQMStore::mergeAndResetMEsRunSummaryCache - Merging objects from run: "
              << run
              << ", stream: " << streamId
              << " module: " << moduleId << std::endl;

  if (LSbasedMode_) {
    return;
  }

  std::string null_str("");
  MonitorElement proto(&null_str, null_str, run, streamId, moduleId);
  // Since this accesses the data, the operation must be
  // be locked.
  std::lock_guard<std::mutex> guard(book_mutex_);

  auto e = data_.end();
  auto i = data_.lower_bound(proto);
  while (i != e) {
    if (i->data_.run != run
        || i->data_.streamId != streamId
        || i->data_.moduleId != moduleId)
      break;

    // Handle Run-based histograms only.
    if (i->getLumiFlag() || LSbasedMode_) {
      ++i;
      continue;
    }

    // don't call the copy constructor
    // we are just searching for a global histogram - a copy is not necessary
    MonitorElement global_me(*i, MonitorElementNoCloneTag());
    global_me.globalize();

    auto me = data_.find(global_me);
    if (me != data_.end()) {
      if (verbose_ > 1)
          std::cout << "Found global Object, using it --> " << me->getFullname() << std::endl;

      //don't take any action if the ME is an INT || FLOAT || STRING
      if(me->kind() >= MonitorElement::DQM_KIND_TH1F)
    {
      if(me->getTH1()->CanExtendAllAxes() && i->getTH1()->CanExtendAllAxes()) {
        TList list;
        list.Add(i->getTH1());
        if( -1 == me->getTH1()->Merge(&list)) {
          std::cout << "mergeAndResetMEsRunSummaryCache: Failed to merge DQM element "<<me->getFullname();
        }
      }
      else {
            if (i->getTH1()->GetEntries())
          me->getTH1()->Add(i->getTH1());
          }
    }
    } else {
      if (verbose_ > 1)
        std::cout << "No global Object found. " << std::endl;
      std::pair<std::set<MonitorElement>::const_iterator, bool> gme;

      // this makes an actual and a single copy with Clone()'ed th1
      MonitorElement actual_global_me(*i);
      actual_global_me.globalize();
      actual_global_me.markToDelete();
      gme = data_.insert(std::move(actual_global_me));
      assert(gme.second);
    }
    // TODO(rovere): eventually reset the local object and mark it as reusable??
    ++i;
  }
}

void DQMStore::mergeAndResetMEsLuminositySummaryCache(uint32_t run,
                              uint32_t lumi,
                              uint32_t streamId,
                              uint32_t moduleId) {
  if (verbose_ > 1)
    std::cout << "DQMStore::mergeAndResetMEsLuminositySummaryCache - Merging objects from run: "
              << run <<     " lumi: " << lumi
              << ", stream: " << streamId
              << " module: " << moduleId << std::endl;
  std::string null_str("");
  MonitorElement proto(&null_str, null_str, run, streamId, moduleId);

  // Since this accesses the data, the operation must be
  // be locked.
  std::lock_guard<std::mutex> guard(book_mutex_);

  auto e = data_.end();
  auto i = data_.lower_bound(proto);

  while (i != e) {
    if (i->data_.run != run
        || i->data_.streamId != streamId
        || i->data_.moduleId != moduleId)
      break;

    // Handle LS-based histograms only.
    if (not (i->getLumiFlag() || LSbasedMode_)) {
      ++i;
      continue;
    }

    MonitorElement global_me(*i, MonitorElementNoCloneTag());
    global_me.globalize();
    global_me.setLumi(lumi);
    auto me = data_.find(global_me);
    if (me != data_.end()) {
      if (verbose_ > 1)
          std::cout << "Found global Object, using it --> " << me->getFullname() << std::endl;

      //don't take any action if the ME is an INT || FLOAT || STRING
      if(me->kind() >= MonitorElement::DQM_KIND_TH1F)
    {
      if(me->getTH1()->CanExtendAllAxes() && i->getTH1()->CanExtendAllAxes()) {
        TList list;
        list.Add(i->getTH1());
        if( -1 == me->getTH1()->Merge(&list)) {
          std::cout << "mergeAndResetMEsLuminositySummaryCache: Failed to merge DQM element "<<me->getFullname();
        }
      }
      else {
            if (i->getTH1()->GetEntries())
          me->getTH1()->Add(i->getTH1());
          }
    }
    } else {
      if (verbose_ > 1)
        std::cout << "No global Object found. " << std::endl;
      std::pair<std::set<MonitorElement>::const_iterator, bool> gme;

      // this makes an actual and a single copy with Clone()'ed th1
      MonitorElement actual_global_me(*i);
      actual_global_me.globalize();
      actual_global_me.setLumi(lumi);
      actual_global_me.markToDelete();
      gme = data_.insert(std::move(actual_global_me));
      assert(gme.second);
    }
    // make the ME reusable for the next LS
    const_cast<MonitorElement*>(&*i)->Reset();
    ++i;
  }
}

DQMStore::DQMStore(const edm::ParameterSet &pset, edm::ActivityRegistry& ar)
  : verbose_ (1),
    verboseQT_ (1),
    reset_ (false),
    collateHistograms_ (false),
    enableMultiThread_(false),
    forceResetOnBeginLumi_(false),
    readSelectedDirectory_ (""),
    run_(0),
    streamId_(0),
    moduleId_(0),
    stream_(nullptr),
    pwd_ (""),
    ibooker_(nullptr),
    igetter_(nullptr)
{
  if (!ibooker_)
    ibooker_ = new DQMStore::IBooker(this);
  if (!igetter_)
    igetter_ = new DQMStore::IGetter(this);
  initializeFrom(pset);

  ar.preallocateSignal_.connect([this](edm::service::SystemBounds const& iBounds) {
      if(iBounds.maxNumberOfStreams() > 1 ) {
    enableMultiThread_ = true;
      }
    });
  if(pset.getUntrackedParameter<bool>("forceResetOnBeginRun",false)) {
    ar.watchPostSourceRun(this,&DQMStore::forceReset);
  }
  if(pset.getUntrackedParameter<bool>("forceResetOnBeginLumi",false) && enableMultiThread_ == false) {
    forceResetOnBeginLumi_ = true;
    ar.watchPostSourceLumi(this,&DQMStore::forceReset);
  }
  ar.watchPostGlobalBeginLumi(this, &DQMStore::postGlobalBeginLumi);
}

DQMStore::DQMStore(const edm::ParameterSet &pset)
  : verbose_ (1),
    verboseQT_ (1),
    reset_ (false),
    collateHistograms_ (false),
    enableMultiThread_(false),
    readSelectedDirectory_ (""),
    run_(0),
    streamId_(0),
    moduleId_(0),
    stream_(nullptr),
    pwd_ (""),
    ibooker_(nullptr),
    igetter_(nullptr)
{
  if (!ibooker_)
    ibooker_ = new DQMStore::IBooker(this);
  if (!igetter_)
    igetter_ = new DQMStore::IGetter(this);
  initializeFrom(pset);
}

DQMStore::~DQMStore()
{
  for (auto & qtest : qtests_)
    delete qtest.second;

  for (auto & qtestspec : qtestspecs_)
    delete qtestspec.first;

  if (stream_)
    stream_->close();
  delete stream_;
}

void
DQMStore::initializeFrom(const edm::ParameterSet& pset) {
  makeDirectory("");
  reset();

  // set steerable parameters
  verbose_ = pset.getUntrackedParameter<int>("verbose", 0);
  if (verbose_ > 0)
    std::cout << "DQMStore: verbosity set to " << verbose_ << std::endl;

  verboseQT_ = pset.getUntrackedParameter<int>("verboseQT", 0);
  if (verbose_ > 0)
    std::cout << "DQMStore: QTest verbosity set to " << verboseQT_ << std::endl;

  collateHistograms_ = pset.getUntrackedParameter<bool>("collateHistograms", false);
  if (collateHistograms_)
    std::cout << "DQMStore: histogram collation is enabled\n";

  enableMultiThread_ = pset.getUntrackedParameter<bool>("enableMultiThread", false);
  if (enableMultiThread_)
    std::cout << "DQMStore: MultiThread option is enabled\n";

  LSbasedMode_ = pset.getUntrackedParameter<bool>("LSbasedMode", false);
   if (LSbasedMode_)
     std::cout << "DQMStore: LSbasedMode option is enabled\n";

  std::string ref = pset.getUntrackedParameter<std::string>("referenceFileName", "");
  if (! ref.empty())
  {
    std::cout << "DQMStore: using reference file '" << ref << "'\n";
    readFile(ref, true, "", s_referenceDirName, StripRunDirs, false);
  }

  initQCriterion<Comp2RefChi2>(qalgos_);
  initQCriterion<Comp2Ref2DChi2>(qalgos_);
  initQCriterion<Comp2RefKolmogorov>(qalgos_);
  initQCriterion<ContentsXRange>(qalgos_);
  initQCriterion<ContentsYRange>(qalgos_);
  initQCriterion<MeanWithinExpected>(qalgos_);
  initQCriterion<Comp2RefEqualH>(qalgos_);
  initQCriterion<DeadChannel>(qalgos_);
  initQCriterion<NoisyChannel>(qalgos_);
  initQCriterion<ContentsWithinExpected>(qalgos_);
  initQCriterion<CompareToMedian>(qalgos_);
  initQCriterion<CompareLastFilledBin>(qalgos_);
  initQCriterion<CheckVariance>(qalgos_);

  scaleFlag_ = pset.getUntrackedParameter<double>("ScalingFlag", 0.0);
  if (verbose_ > 0)
    std::cout << "DQMStore: Scaling Flag set to " << scaleFlag_ << std::endl;
}

/* Generic method to do a backtrace and print it to stdout. It is
 customised to properly get the routine that called the booking of the
 histograms, which, following the usual stack, is at position 4. The
 name of the calling function is properly demangled and the original
 shared library including this function is also printed. For a more
 detailed explanation of the routines involved, see here:
 http://www.gnu.org/software/libc/manual/html_node/Backtraces.html
 http://gcc.gnu.org/onlinedocs/libstdc++/manual/ext_demangling.html.*/

void
DQMStore::print_trace (const std::string &dir, const std::string &name)
{
  // the access to the member stream_ is implicitely protected against
  // concurrency problems because the print_trace method is always called behind
  // a lock (see bookTransaction).
  if (!stream_)
    stream_ = new std::ofstream("histogramBookingBT.log");

  void *array[10];
  size_t size;
  char **strings;
  int r=0;
  lat::RegexpMatch m;
  m.reset();

  size = backtrace (array, 10);
  strings = backtrace_symbols (array, size);

  size_t level = 1;
  char * demangled = nullptr;
  for (; level < size; level++) {
    if (!s_rxtrace.match(strings[level], 0, 0, &m)) continue;
    demangled = abi::__cxa_demangle(m.matchString(strings[level], 2).c_str(), nullptr, nullptr, &r);
    if (!demangled) continue;
    if (!s_rxself.match(demangled, 0, 0)) break;
    free(demangled);
    demangled = nullptr;
  }

  if (demangled != nullptr) {
    *stream_ << "\"" << dir << "/"
           << name << "\" "
           << (r ? m.matchString(strings[level], 2) : demangled) << " "
           << m.matchString(strings[level], 1) << "\n";
    free(demangled);
  } else {
    *stream_ << "Skipping "<< dir << "/" << name
           << " with stack size " << size << "\n";
  }

  /* In this case print the full stack trace, up to main or to the
   * maximum stack size, i.e. 10. */
  if (verbose_ > 4 || demangled == nullptr)
  {
    size_t i;
    m.reset();

    for (i = 0; i < size; i++)
      if (s_rxtrace.match(strings[i], 0, 0, &m))
      {
        char * demangled = abi::__cxa_demangle(m.matchString(strings[i], 2).c_str(), nullptr, nullptr, &r);
        *stream_ << "\t\t" << i << "/" << size << " "
               << (r ? m.matchString(strings[i], 2) : demangled) << " "
               << m.matchString(strings[i], 1) << std::endl;
        free (demangled);
      }
  }
  free (strings);
}

void
DQMStore::setVerbose(unsigned /* level */)
{ return; }

const std::string &
DQMStore::pwd() const
{ return pwd_; }

void
DQMStore::cd()
{ setCurrentFolder(""); }

void
DQMStore::cd(const std::string &subdir)
{
  std::string clean;
  const std::string *cleaned = nullptr;
  cleanTrailingSlashes(subdir, clean, cleaned);

  if (! dirExists(*cleaned))
    raiseDQMError("DQMStore", "Cannot 'cd' into non-existent directory '%s'",
                  cleaned->c_str());

  setCurrentFolder(*cleaned);
}

void
DQMStore::setCurrentFolder(const std::string &fullpath)
{
  std::string clean;
  const std::string *cleaned = nullptr;
  cleanTrailingSlashes(fullpath, clean, cleaned);
  makeDirectory(*cleaned);
  pwd_ = *cleaned;
}

void
DQMStore::goUp()
{
  size_t pos = pwd_.rfind('/');
  if (pos == std::string::npos)
    setCurrentFolder("");
  else
    setCurrentFolder(pwd_.substr(0, pos));
}

// -------------------------------------------------------------------
void
DQMStore::makeDirectory(const std::string &path)
{
  std::string prev;
  std::string subdir;
  std::string name;
  prev.reserve(path.size());
  subdir.reserve(path.size());
  name.reserve(path.size());
  size_t prevname = 0;
  size_t slash = 0;

  while (true)
  {
    // Create this subdirectory component.
    subdir.clear();
    subdir.append(path, 0, slash);
    name.clear();
    name.append(subdir, prevname, std::string::npos);
    if (! prev.empty() && findObject(prev, name))
      raiseDQMError("DQMStore", "Attempt to create subdirectory '%s'"
                    " which already exists as a monitor element",
                    subdir.c_str());

    if (! dirs_.count(subdir))
      dirs_.insert(subdir);

    // Stop if we've reached the end (including possibly a trailing slash).
    if (slash+1 >= path.size())
      break;

    // Find the next slash, making sure we progress.  If reach the end,
    // process the last path component; the next loop round will terminate.
    prevname = slash ? slash+1 : slash;
    prev = subdir;
    if ((slash = path.find('/', ++slash)) == std::string::npos)
      slash = path.size();
  }
}

bool
DQMStore::dirExists(const std::string &path) const
{ return dirs_.count(path) > 0; }

template <class HISTO, class COLLATE>
MonitorElement *
DQMStore::book_(const std::string &dir, const std::string &name,
                const char *context, int kind,
                HISTO *h, COLLATE collate)
{
  assert(name.find('/') == std::string::npos);
  if (verbose_ > 3)
    print_trace(dir, name);
  std::string path;
  mergePath(path, dir, name);

  // Put us in charge of h.
  h->SetDirectory(nullptr);

  // Check if the request monitor element already exists.
  MonitorElement *me = findObject(dir, name, run_, 0, streamId_, moduleId_);
  if (me)
  {
    if (collateHistograms_)
    {
      collate(me, h, verbose_);
      delete h;
      return me;
    }
    else
    {
      if (verbose_ > 1)
        std::cout << "DQMStore: "
                  << context << ": monitor element '"
                  << path << "' already exists, collating" << std::endl;
      me->Reset();
      collate(me, h, verbose_);
      delete h;
      return me;
    }
  }
  else
  {
    // Create and initialise core object.
    assert(dirs_.count(dir));
    MonitorElement proto(&*dirs_.find(dir), name, run_, streamId_, moduleId_);
    me = const_cast<MonitorElement &>(*data_.insert(std::move(proto)).first)
      .initialise((MonitorElement::Kind)kind, h);

    // Initialise quality test information.
    auto qi = qtestspecs_.begin();
    auto qe = qtestspecs_.end();
    for ( ; qi != qe; ++qi)
    {
        if ( qi->first->match(path) )
                me->addQReport(qi->second);
    }

    // If we just booked a (plain) MonitorElement, and there is a reference
    // MonitorElement with the same name, link the two together.
    // The other direction is handled by the extract method.
    std::string refdir;
    refdir.reserve(s_referenceDirName.size() + dir.size() + 1);
    refdir += s_referenceDirName;
    refdir += '/';
    refdir += dir;
    MonitorElement* referenceME = findObject(refdir, name);
    if (referenceME) {
      // We have booked a new MonitorElement with a specific dir and name.
      // Then, if we can find the corresponding MonitorElement in the reference
      // dir we assign the object_ of the reference MonitorElement to the
      // reference_ property of our new MonitorElement.
      me->data_.flags |= DQMNet::DQM_PROP_HAS_REFERENCE;
      me->reference_ = referenceME->object_;
    }

    // Return the monitor element.
    return me;
  }
}

MonitorElement *
DQMStore::book_(const std::string &dir,
                const std::string &name,
                const char *context)
{
  assert(name.find('/') == std::string::npos);
  if (verbose_ > 3)
    print_trace(dir, name);

  // Check if the request monitor element already exists.
  if (MonitorElement *me = findObject(dir, name, run_, 0, streamId_, moduleId_))
  {
    if (verbose_ > 1)
    {
      std::string path;
      mergePath(path, dir, name);

      std::cout << "DQMStore: "
                << context << ": monitor element '"
                << path << "' already exists, resetting" << std::endl;
    }
    me->Reset();
    return me;
  }
  else
  {
    // Create it and return for initialisation.
    assert(dirs_.count(dir));
    MonitorElement proto(&*dirs_.find(dir), name, run_, streamId_, moduleId_);
    return &const_cast<MonitorElement &>(*data_.insert(std::move(proto)).first);
  }
}

// -------------------------------------------------------------------
MonitorElement *
DQMStore::bookInt_(const std::string &dir, const std::string &name)
{
  if (collateHistograms_)
  {
    if (MonitorElement *me = findObject(dir, name, run_, 0, streamId_, moduleId_))
    {
      me->Fill(0);
      return me;
    }
  }

  return book_(dir, name, "bookInt")
    ->initialise(MonitorElement::DQM_KIND_INT);
}

MonitorElement *
DQMStore::bookInt(const char *name)
{ return bookInt_(pwd_, name); }

MonitorElement *
DQMStore::bookInt(const std::string &name)
{
  return bookInt_(pwd_, name);
}

// -------------------------------------------------------------------
MonitorElement *
DQMStore::bookFloat_(const std::string &dir, const std::string &name)
{
  if (collateHistograms_)
  {
    if (MonitorElement *me = findObject(dir, name, run_, 0, streamId_, moduleId_))
    {
      me->Fill(0.);
      return me;
    }
  }

  return book_(dir, name, "bookFloat")
    ->initialise(MonitorElement::DQM_KIND_REAL);
}

MonitorElement *
DQMStore::bookFloat(const char *name)
{ return bookFloat_(pwd_, name); }

MonitorElement *
DQMStore::bookFloat(const std::string &name)
{
  return bookFloat_(pwd_, name);
}

// -------------------------------------------------------------------
MonitorElement *
DQMStore::bookString_(const std::string &dir,
                      const std::string &name,
                      const std::string &value)
{
  if (collateHistograms_)
  {
    if (MonitorElement *me = findObject(dir, name, run_, 0, streamId_, moduleId_))
      return me;
  }

  return book_(dir, name, "bookString")
    ->initialise(MonitorElement::DQM_KIND_STRING, value);
}

MonitorElement *
DQMStore::bookString(const char *name, const char *value)
{ return bookString_(pwd_, name, value); }

MonitorElement *
DQMStore::bookString(const std::string &name, const std::string &value)
{
  return bookString_(pwd_, name, value);
}

// -------------------------------------------------------------------
MonitorElement *
DQMStore::book1D_(const std::string &dir, const std::string &name, TH1F *h)
{
  return book_(dir, name, "book1D", MonitorElement::DQM_KIND_TH1F, h, collate1D);
}

MonitorElement *
DQMStore::book1S_(const std::string &dir, const std::string &name, TH1S *h)
{
  return book_(dir, name, "book1S", MonitorElement::DQM_KIND_TH1S, h, collate1S);
}

MonitorElement *
DQMStore::book1DD_(const std::string &dir, const std::string &name, TH1D *h)
{
  return book_(dir, name, "book1DD", MonitorElement::DQM_KIND_TH1D, h, collate1DD);
}

MonitorElement *
DQMStore::book1D(const char *name, const char *title,
                 int nchX, double lowX, double highX)
{
  return book1D_(pwd_, name, new TH1F(name, title, nchX, lowX, highX));
}

MonitorElement *
DQMStore::book1D(const std::string &name, const std::string &title,
                 int nchX, double lowX, double highX)
{
  return book1D_(pwd_, name, new TH1F(name.c_str(), title.c_str(), nchX, lowX, highX));
}

MonitorElement *
DQMStore::book1S(const char *name, const char *title,
                 int nchX, double lowX, double highX)
{
  return book1S_(pwd_, name, new TH1S(name, title, nchX, lowX, highX));
}

MonitorElement *
DQMStore::book1S(const std::string &name, const std::string &title,
                 int nchX, double lowX, double highX)
{
  return book1S_(pwd_, name, new TH1S(name.c_str(), title.c_str(), nchX, lowX, highX));
}

MonitorElement *
DQMStore::book1DD(const char *name, const char *title,
                  int nchX, double lowX, double highX)
{
  return book1DD_(pwd_, name, new TH1D(name, title, nchX, lowX, highX));
}

MonitorElement *
DQMStore::book1DD(const std::string &name, const std::string &title,
                  int nchX, double lowX, double highX)
{
  return book1DD_(pwd_, name, new TH1D(name.c_str(), title.c_str(), nchX, lowX, highX));
}

MonitorElement *
DQMStore::book1D(const char *name, const char *title,
                 int nchX, const float *xbinsize)
{
  return book1D_(pwd_, name, new TH1F(name, title, nchX, xbinsize));
}

MonitorElement *
DQMStore::book1D(const std::string &name, const std::string &title,
                 int nchX, const float *xbinsize)
{
  return book1D_(pwd_, name, new TH1F(name.c_str(), title.c_str(), nchX, xbinsize));
}

MonitorElement *
DQMStore::book1D(const char *name, TH1F *source)
{
  return book1D_(pwd_, name, static_cast<TH1F *>(source->Clone(name)));
}

MonitorElement *
DQMStore::book1D(const std::string &name, TH1F *source)
{
  return book1D_(pwd_, name, static_cast<TH1F *>(source->Clone(name.c_str())));
}

MonitorElement *
DQMStore::book1S(const char *name, TH1S *source)
{
  return book1S_(pwd_, name, static_cast<TH1S *>(source->Clone(name)));
}

MonitorElement *
DQMStore::book1S(const std::string &name, TH1S *source)
{
  return book1S_(pwd_, name, static_cast<TH1S *>(source->Clone(name.c_str())));
}

MonitorElement *
DQMStore::book1DD(const char *name, TH1D *source)
{
  return book1DD_(pwd_, name, static_cast<TH1D *>(source->Clone(name)));
}

MonitorElement *
DQMStore::book1DD(const std::string &name, TH1D *source)
{
  return book1DD_(pwd_, name, static_cast<TH1D *>(source->Clone(name.c_str())));
}

// -------------------------------------------------------------------
MonitorElement *
DQMStore::book2D_(const std::string &dir, const std::string &name, TH2F *h)
{
  return book_(dir, name, "book2D", MonitorElement::DQM_KIND_TH2F, h, collate2D);
}

MonitorElement *
DQMStore::book2S_(const std::string &dir, const std::string &name, TH2S *h)
{
  return book_(dir, name, "book2S", MonitorElement::DQM_KIND_TH2S, h, collate2S);
}

MonitorElement *
DQMStore::book2DD_(const std::string &dir, const std::string &name, TH2D *h)
{
  return book_(dir, name, "book2DD", MonitorElement::DQM_KIND_TH2D, h, collate2DD);
}

MonitorElement *
DQMStore::book2D(const char *name, const char *title,
                 int nchX, double lowX, double highX,
                 int nchY, double lowY, double highY)
{
  return book2D_(pwd_, name, new TH2F(name, title,
                                     nchX, lowX, highX,
                                     nchY, lowY, highY));
}

MonitorElement *
DQMStore::book2D(const std::string &name, const std::string &title,
                 int nchX, double lowX, double highX,
                 int nchY, double lowY, double highY)
{
  return book2D_(pwd_, name, new TH2F(name.c_str(), title.c_str(),
                                     nchX, lowX, highX,
                                     nchY, lowY, highY));
}

MonitorElement *
DQMStore::book2S(const char *name, const char *title,
                 int nchX, double lowX, double highX,
                 int nchY, double lowY, double highY)
{
  return book2S_(pwd_, name, new TH2S(name, title,
                                     nchX, lowX, highX,
                                     nchY, lowY, highY));
}

MonitorElement *
DQMStore::book2S(const std::string &name, const std::string &title,
                 int nchX, double lowX, double highX,
                 int nchY, double lowY, double highY)
{
  return book2S_(pwd_, name, new TH2S(name.c_str(), title.c_str(),
                                     nchX, lowX, highX,
                                     nchY, lowY, highY));
}

MonitorElement *
DQMStore::book2DD(const char *name, const char *title,
                  int nchX, double lowX, double highX,
                  int nchY, double lowY, double highY)
{
  return book2DD_(pwd_, name, new TH2D(name, title,
                                      nchX, lowX, highX,
                                      nchY, lowY, highY));
}

MonitorElement *
DQMStore::book2DD(const std::string &name, const std::string &title,
                  int nchX, double lowX, double highX,
                  int nchY, double lowY, double highY)
{
  return book2DD_(pwd_, name, new TH2D(name.c_str(), title.c_str(),
                                      nchX, lowX, highX,
                                      nchY, lowY, highY));
}

MonitorElement *
DQMStore::book2D(const char *name, const char *title,
                 int nchX, const float *xbinsize, int nchY, const float *ybinsize)
{
  return book2D_(pwd_, name, new TH2F(name, title,
                                     nchX, xbinsize, nchY, ybinsize));
}

MonitorElement *
DQMStore::book2D(const std::string &name, const std::string &title,
                 int nchX, const float *xbinsize, int nchY, const float *ybinsize)
{
  return book2D_(pwd_, name, new TH2F(name.c_str(), title.c_str(),
                                     nchX, xbinsize, nchY, ybinsize));
}

MonitorElement *
DQMStore::book2S(const char *name, const char *title,
                 int nchX, const float *xbinsize, int nchY, const float *ybinsize)
{
  return book2S_(pwd_, name, new TH2S(name, title,
                                     nchX, xbinsize, nchY, ybinsize));
}

MonitorElement *
DQMStore::book2S(const std::string &name, const std::string &title,
                 int nchX, const float *xbinsize, int nchY, const float *ybinsize)
{
  return book2S_(pwd_, name, new TH2S(name.c_str(), title.c_str(),
                                     nchX, xbinsize, nchY, ybinsize));
}

MonitorElement *
DQMStore::book2D(const char *name, TH2F *source)
{
  return book2D_(pwd_, name, static_cast<TH2F *>(source->Clone(name)));
}

MonitorElement *
DQMStore::book2D(const std::string &name, TH2F *source)
{
  return book2D_(pwd_, name, static_cast<TH2F *>(source->Clone(name.c_str())));
}

MonitorElement *
DQMStore::book2S(const char *name, TH2S *source)
{
  return book2S_(pwd_, name, static_cast<TH2S *>(source->Clone(name)));
}

MonitorElement *
DQMStore::book2S(const std::string &name, TH2S *source)
{
  return book2S_(pwd_, name, static_cast<TH2S *>(source->Clone(name.c_str())));
}

MonitorElement *
DQMStore::book2DD(const char *name, TH2D *source)
{
  return book2DD_(pwd_, name, static_cast<TH2D *>(source->Clone(name)));
}

MonitorElement *
DQMStore::book2DD(const std::string &name, TH2D *source)
{
  return book2DD_(pwd_, name, static_cast<TH2D *>(source->Clone(name.c_str())));
}

// -------------------------------------------------------------------
MonitorElement *
DQMStore::book3D_(const std::string &dir, const std::string &name, TH3F *h)
{
  return book_(dir, name, "book3D", MonitorElement::DQM_KIND_TH3F, h, collate3D);
}

MonitorElement *
DQMStore::book3D(const char *name, const char *title,
                 int nchX, double lowX, double highX,
                 int nchY, double lowY, double highY,
                 int nchZ, double lowZ, double highZ)
{
  return book3D_(pwd_, name, new TH3F(name, title,
                                     nchX, lowX, highX,
                                     nchY, lowY, highY,
                                     nchZ, lowZ, highZ));
}

MonitorElement *
DQMStore::book3D(const std::string &name, const std::string &title,
                 int nchX, double lowX, double highX,
                 int nchY, double lowY, double highY,
                 int nchZ, double lowZ, double highZ)
{
  return book3D_(pwd_, name, new TH3F(name.c_str(), title.c_str(),
                                     nchX, lowX, highX,
                                     nchY, lowY, highY,
                                     nchZ, lowZ, highZ));
}

MonitorElement *
DQMStore::book3D(const char *name, TH3F *source)
{
  return book3D_(pwd_, name, static_cast<TH3F *>(source->Clone(name)));
}

MonitorElement *
DQMStore::book3D(const std::string &name, TH3F *source)
{
  return book3D_(pwd_, name, static_cast<TH3F *>(source->Clone(name.c_str())));
}

// -------------------------------------------------------------------
MonitorElement *
DQMStore::bookProfile_(const std::string &dir, const std::string &name, TProfile *h)
{
  return book_(dir, name, "bookProfile",
               MonitorElement::DQM_KIND_TPROFILE,
               h, collateProfile);
}

MonitorElement *
DQMStore::bookProfile(const char *name, const char *title,
                      int nchX, double lowX, double highX,
                      int /* nchY */, double lowY, double highY,
                      const char *option /* = "s" */)
{
  return bookProfile_(pwd_, name, new TProfile(name, title,
                                              nchX, lowX, highX,
                                              lowY, highY,
                                              option));
}

MonitorElement *
DQMStore::bookProfile(const std::string &name, const std::string &title,
                      int nchX, double lowX, double highX,
                      int /* nchY */, double lowY, double highY,
                      const char *option /* = "s" */)
{
  return bookProfile_(pwd_, name, new TProfile(name.c_str(), title.c_str(),
                                              nchX, lowX, highX,
                                              lowY, highY,
                                              option));
}

MonitorElement *
DQMStore::bookProfile(const char *name, const char *title,
                      int nchX, double lowX, double highX,
                      double lowY, double highY,
                      const char *option /* = "s" */)
{
  return bookProfile_(pwd_, name, new TProfile(name, title,
                                              nchX, lowX, highX,
                                              lowY, highY,
                                              option));
}

MonitorElement *
DQMStore::bookProfile(const std::string &name, const std::string &title,
                      int nchX, double lowX, double highX,
                      double lowY, double highY,
                      const char *option /* = "s" */)
{
  return bookProfile_(pwd_, name, new TProfile(name.c_str(), title.c_str(),
                                              nchX, lowX, highX,
                                              lowY, highY,
                                              option));
}

MonitorElement *
DQMStore::bookProfile(const char *name, const char *title,
                      int nchX, const double *xbinsize,
                      int /* nchY */, double lowY, double highY,
                      const char *option /* = "s" */)
{
  return bookProfile_(pwd_, name, new TProfile(name, title,
                                              nchX, xbinsize,
                                              lowY, highY,
                                              option));
}

MonitorElement *
DQMStore::bookProfile(const std::string &name, const std::string &title,
                      int nchX, const double *xbinsize,
                      int /* nchY */, double lowY, double highY,
                      const char *option /* = "s" */)
{
  return bookProfile_(pwd_, name, new TProfile(name.c_str(), title.c_str(),
                                              nchX, xbinsize,
                                              lowY, highY,
                                              option));
}

MonitorElement *
DQMStore::bookProfile(const char *name, const char *title,
                      int nchX, const double *xbinsize,
                      double lowY, double highY,
                      const char *option /* = "s" */)
{
  return bookProfile_(pwd_, name, new TProfile(name, title,
                                              nchX, xbinsize,
                                              lowY, highY,
                                              option));
}

MonitorElement *
DQMStore::bookProfile(const std::string &name, const std::string &title,
                      int nchX, const double *xbinsize,
                      double lowY, double highY,
                      const char *option /* = "s" */)
{
  return bookProfile_(pwd_, name, new TProfile(name.c_str(), title.c_str(),
                                              nchX, xbinsize,
                                              lowY, highY,
                                              option));
}

MonitorElement *
DQMStore::bookProfile(const char *name, TProfile *source)
{
  return bookProfile_(pwd_, name, static_cast<TProfile *>(source->Clone(name)));
}

MonitorElement *
DQMStore::bookProfile(const std::string &name, TProfile *source)
{
  return bookProfile_(pwd_, name, static_cast<TProfile *>(source->Clone(name.c_str())));
}

// -------------------------------------------------------------------
MonitorElement *
DQMStore::bookProfile2D_(const std::string &dir, const std::string &name, TProfile2D *h)
{
  return book_(dir, name, "bookProfile2D",
               MonitorElement::DQM_KIND_TPROFILE2D,
               h, collateProfile2D);
}

MonitorElement *
DQMStore::bookProfile2D(const char *name, const char *title,
                        int nchX, double lowX, double highX,
                        int nchY, double lowY, double highY,
                        int /* nchZ */, double lowZ, double highZ,
                        const char *option /* = "s" */)
{
  return bookProfile2D_(pwd_, name, new TProfile2D(name, title,
                                                  nchX, lowX, highX,
                                                  nchY, lowY, highY,
                                                  lowZ, highZ,
                                                  option));
}

MonitorElement *
DQMStore::bookProfile2D(const std::string &name, const std::string &title,
                        int nchX, double lowX, double highX,
                        int nchY, double lowY, double highY,
                        int /* nchZ */, double lowZ, double highZ,
                        const char *option /* = "s" */)
{
  return bookProfile2D_(pwd_, name, new TProfile2D(name.c_str(), title.c_str(),
                                                  nchX, lowX, highX,
                                                  nchY, lowY, highY,
                                                  lowZ, highZ,
                                                  option));
}

MonitorElement *
DQMStore::bookProfile2D(const char *name, const char *title,
                        int nchX, double lowX, double highX,
                        int nchY, double lowY, double highY,
                        double lowZ, double highZ,
                        const char *option /* = "s" */)
{
  return bookProfile2D_(pwd_, name, new TProfile2D(name, title,
                                                  nchX, lowX, highX,
                                                  nchY, lowY, highY,
                                                  lowZ, highZ,
                                                  option));
}

MonitorElement *
DQMStore::bookProfile2D(const std::string &name, const std::string &title,
                        int nchX, double lowX, double highX,
                        int nchY, double lowY, double highY,
                        double lowZ, double highZ,
                        const char *option /* = "s" */)
{
  return bookProfile2D_(pwd_, name, new TProfile2D(name.c_str(), title.c_str(),
                                                  nchX, lowX, highX,
                                                  nchY, lowY, highY,
                                                  lowZ, highZ,
                                                  option));
}

MonitorElement *
DQMStore::bookProfile2D(const char *name, TProfile2D *source)
{
  return bookProfile2D_(pwd_, name, static_cast<TProfile2D *>(source->Clone(name)));
}

MonitorElement *
DQMStore::bookProfile2D(const std::string &name, TProfile2D *source)
{
  return bookProfile2D_(pwd_, name, static_cast<TProfile2D *>(source->Clone(name.c_str())));
}

bool
DQMStore::checkBinningMatches(MonitorElement *me, TH1 *h, unsigned verbose)
{
  if (me->getTH1()->GetNbinsX() != h->GetNbinsX()
      || me->getTH1()->GetNbinsY() != h->GetNbinsY()
      || me->getTH1()->GetNbinsZ() != h->GetNbinsZ()
      || me->getTH1()->GetXaxis()->GetXmin() != h->GetXaxis()->GetXmin()
      || me->getTH1()->GetYaxis()->GetXmin() != h->GetYaxis()->GetXmin()
      || me->getTH1()->GetZaxis()->GetXmin() != h->GetZaxis()->GetXmin()
      || me->getTH1()->GetXaxis()->GetXmax() != h->GetXaxis()->GetXmax()
      || me->getTH1()->GetYaxis()->GetXmax() != h->GetYaxis()->GetXmax()
      || me->getTH1()->GetZaxis()->GetXmax() != h->GetZaxis()->GetXmax()
      || !MonitorElement::CheckBinLabels((TAxis*)me->getTH1()->GetXaxis(),(TAxis*)h->GetXaxis())
      || !MonitorElement::CheckBinLabels((TAxis*)me->getTH1()->GetYaxis(),(TAxis*)h->GetYaxis())
      || !MonitorElement::CheckBinLabels((TAxis*)me->getTH1()->GetZaxis(),(TAxis*)h->GetZaxis()) )
  {
    if(verbose > 0)
      std::cout << "*** DQMStore: WARNING:"
        << "checkBinningMatches: different binning - cannot add object '"
        << h->GetName() << "' of type "
        << h->IsA()->GetName() << " to existing ME: '"
        << me->getFullname() << "'\n";
    return false;
  }
  return true;
}

void
DQMStore::collate1D(MonitorElement *me, TH1F *h, unsigned verbose)
{
  if (checkBinningMatches(me,h,verbose))
    me->getTH1F()->Add(h);
}

void
DQMStore::collate1S(MonitorElement *me, TH1S *h, unsigned verbose)
{
  if (checkBinningMatches(me,h,verbose))
    me->getTH1S()->Add(h);
}

void
DQMStore::collate1DD(MonitorElement *me, TH1D *h, unsigned verbose)
{
  if (checkBinningMatches(me,h,verbose))
    me->getTH1D()->Add(h);
}

void
DQMStore::collate2D(MonitorElement *me, TH2F *h, unsigned verbose)
{
  if (checkBinningMatches(me,h,verbose))
    me->getTH2F()->Add(h);
}

void
DQMStore::collate2S(MonitorElement *me, TH2S *h, unsigned verbose)
{
  if (checkBinningMatches(me,h,verbose))
    me->getTH2S()->Add(h);
}

void
DQMStore::collate2DD(MonitorElement *me, TH2D *h, unsigned verbose)
{
  if (checkBinningMatches(me,h,verbose))
    me->getTH2D()->Add(h);
}

void
DQMStore::collate3D(MonitorElement *me, TH3F *h, unsigned verbose)
{
  if (checkBinningMatches(me,h,verbose))
    me->getTH3F()->Add(h);
}

void
DQMStore::collateProfile(MonitorElement *me, TProfile *h, unsigned verbose)
{
  if (checkBinningMatches(me,h,verbose))
  {
    TProfile *meh = me->getTProfile();
    me->addProfiles(h, meh, meh, 1, 1);
  }
}

void
DQMStore::collateProfile2D(MonitorElement *me, TProfile2D *h, unsigned verbose)
{
  if (checkBinningMatches(me,h,verbose))
  {
    TProfile2D *meh = me->getTProfile2D();
    me->addProfiles(h, meh, meh, 1, 1);
  }
}

void
DQMStore::tag(MonitorElement *me, unsigned int myTag)
{
  if (! myTag)
    raiseDQMError("DQMStore", "Attempt to tag monitor element '%s'"
                  " with a zero tag", me->getFullname().c_str());
  if ((me->data_.flags & DQMNet::DQM_PROP_TAGGED) && myTag != me->data_.tag)
    raiseDQMError("DQMStore", "Attempt to tag monitor element '%s'"
                  " twice with multiple tags", me->getFullname().c_str());

  me->data_.tag = myTag;
  me->data_.flags |= DQMNet::DQM_PROP_TAGGED;
}

void
DQMStore::tag(const std::string &path, unsigned int myTag)
{
  std::string dir;
  std::string name;
  splitPath(dir, name, path);

  if (MonitorElement *me = findObject(dir, name))
    tag(me, myTag);
  else
    raiseDQMError("DQMStore", "Attempt to tag non-existent monitor element"
                  " '%s' with tag %u", path.c_str(), myTag);

}

void
DQMStore::tagContents(const std::string &path, unsigned int myTag)
{
  MonitorElement proto(&path, std::string());
  auto e = data_.end();
  auto i = data_.lower_bound(proto);
  for ( ; i != e && path == *i->data_.dirname; ++i)
    tag(const_cast<MonitorElement *>(&*i), myTag);
}

void
DQMStore::tagAllContents(const std::string &path, unsigned int myTag)
{
  std::string clean;
  const std::string *cleaned = nullptr;
  cleanTrailingSlashes(path, clean, cleaned);
  MonitorElement proto(cleaned, std::string());

  // FIXME: WILDCARDS? Old one supported them, but nobody seemed to use them.
  auto e = data_.end();
  auto i = data_.lower_bound(proto);
  while (i != e && isSubdirectory(*cleaned, *i->data_.dirname))
  {
    tag(const_cast<MonitorElement *>(&*i), myTag);
    ++i;
  }
}

std::vector<std::string>
DQMStore::getSubdirs() const
{
  std::vector<std::string> result;
  auto e = dirs_.end();
  auto i = dirs_.find(pwd_);

  // If we didn't find current directory, the tree is empty, so quit.
  if (i == e)
    return result;

  // Skip the current directory and then start looking for immediate
  // subdirectories in the dirs_ list.  Stop when we are no longer in
  // (direct or indirect) subdirectories of pwd_.  Note that we don't
  // "know" which order the set will sort A/B, A/B/C and A/D.
  while (++i != e && isSubdirectory(pwd_, *i))
    if (i->find('/', pwd_.size()+1) == std::string::npos)
      result.push_back(*i);

  return result;
}

std::vector<std::string>
DQMStore::getMEs() const
{
  MonitorElement proto(&pwd_, std::string());
  std::vector<std::string> result;
  auto e = data_.end();
  auto i = data_.lower_bound(proto);
  for ( ; i != e && isSubdirectory(pwd_, *i->data_.dirname); ++i)
    if (pwd_ == *i->data_.dirname)
      result.push_back(i->getName());

  return result;
}

bool
DQMStore::containsAnyMonitorable(const std::string &path) const
{
  MonitorElement proto(&path, std::string());
  auto e = data_.end();
  auto i = data_.lower_bound(proto);
  return (i != e && isSubdirectory(path, *i->data_.dirname));
}

MonitorElement *
DQMStore::get(const std::string &path) const
{
  std::string dir;
  std::string name;
  splitPath(dir, name, path);
  MonitorElement proto(&dir, name);
  auto mepos = data_.find(proto);
  return (mepos == data_.end() ? nullptr
          : const_cast<MonitorElement *>(&*mepos));
}

std::vector<MonitorElement *>
DQMStore::get(unsigned int tag) const
{
  // FIXME: Use reverse map [tag -> path] / [tag -> dir]?
  std::vector<MonitorElement *> result;
  for (auto const & me : data_)
  {
    if ((me.data_.flags & DQMNet::DQM_PROP_TAGGED) && me.data_.tag == tag)
      result.push_back(const_cast<MonitorElement *>(&me));
  }
  return result;
}

std::vector<MonitorElement *>
DQMStore::getContents(const std::string &path) const
{
  std::string clean;
  const std::string *cleaned = nullptr;
  cleanTrailingSlashes(path, clean, cleaned);
  MonitorElement proto(cleaned, std::string());

  std::vector<MonitorElement *> result;
  auto e = data_.end();
  auto i = data_.lower_bound(proto);
  for ( ; i != e && isSubdirectory(*cleaned, *i->data_.dirname); ++i)
    if (*cleaned == *i->data_.dirname)
      result.push_back(const_cast<MonitorElement *>(&*i));

  return result;
}

std::vector<MonitorElement *>
DQMStore::getContents(const std::string &path, unsigned int tag) const
{
  std::string clean;
  const std::string *cleaned = nullptr;
  cleanTrailingSlashes(path, clean, cleaned);
  MonitorElement proto(cleaned, std::string());

  std::vector<MonitorElement *> result;
  auto e = data_.end();
  auto i = data_.lower_bound(proto);
  for ( ; i != e && isSubdirectory(*cleaned, *i->data_.dirname); ++i)
    if (*cleaned == *i->data_.dirname
        && (i->data_.flags & DQMNet::DQM_PROP_TAGGED)
        && i->data_.tag == tag)
      result.push_back(const_cast<MonitorElement *>(&*i));

  return result;
}

void
DQMStore::getContents(std::vector<std::string> &into, bool showContents /* = true */) const
{
  into.clear();
  into.reserve(dirs_.size());

  auto me = data_.end();
  auto di = dirs_.begin();
  auto de = dirs_.end();
  for ( ; di != de; ++di)
  {
    MonitorElement proto(&*di, std::string());
    auto mi = data_.lower_bound(proto);
    auto m = mi;
    size_t sz = di->size() + 2;
    size_t nfound = 0;
    for ( ; m != me && isSubdirectory(*di, *m->data_.dirname); ++m)
      if (*di == *m->data_.dirname)
      {
        sz += m->data_.objname.size() + 1;
        ++nfound;
      }

    if (! nfound)
      continue;

    auto istr
      = into.insert(into.end(), std::string());

    if (showContents)
    {
      istr->reserve(sz);

      *istr += *di;
      *istr += ':';
      for (sz = 0; mi != m; ++mi)
      {
        if (*di != *mi->data_.dirname)
          continue;

        if (sz > 0)
          *istr += ',';

        *istr += mi->data_.objname;
        ++sz;
      }
    }
    else
    {
      istr->reserve(di->size() + 2);
      *istr += *di;
      *istr += ':';
    }
  }
}

MonitorElement *
DQMStore::findObject(const std::string &dir,
                     const std::string &name,
                     const uint32_t run /* = 0 */,
                     const uint32_t lumi /* = 0 */,
                     const uint32_t streamId /* = 0 */,
                     const uint32_t moduleId /* = 0 */) const
{
  if (dir.find_first_not_of(s_safe) != std::string::npos)
    raiseDQMError("DQMStore", "Monitor element path name '%s' uses"
                  " unacceptable characters", dir.c_str());
  if (name.find_first_not_of(s_safe) != std::string::npos)
    raiseDQMError("DQMStore", "Monitor element path name '%s' uses"
                  " unacceptable characters", name.c_str());

  MonitorElement proto;
  proto.data_.dirname  = &dir;
  proto.data_.objname  = name;
  proto.data_.run      = run;
  proto.data_.lumi     = lumi;
  proto.data_.streamId = streamId;
  proto.data_.moduleId = moduleId;

  auto mepos = data_.find(proto);
  return (mepos == data_.end() ? nullptr
          : const_cast<MonitorElement *>(&*mepos));
}

void
DQMStore::getAllTags(std::vector<std::string> &into) const
{
  into.clear();
  into.reserve(dirs_.size());

  auto me = data_.end();
  auto di = dirs_.begin();
  auto de = dirs_.end();
  char tagbuf[32]; // more than enough for '/' and up to 10 digits

  for ( ; di != de; ++di)
  {
    MonitorElement proto(&*di, std::string());
    auto mi = data_.lower_bound(proto);
    auto m = mi;
    size_t sz = di->size() + 2;
    size_t nfound = 0;
    for ( ; m != me && isSubdirectory(*di, *m->data_.dirname); ++m)
      if (*di == *m->data_.dirname && (m->data_.flags & DQMNet::DQM_PROP_TAGGED))
      {
        // the tags count for '/' + up to 10 digits, otherwise ',' + ME name
        sz += 1 + m->data_.objname.size() + 11;
        ++nfound;
      }

    if (! nfound)
      continue;

    auto istr
      = into.insert(into.end(), std::string());

    istr->reserve(sz);

    *istr += *di;
    *istr += ':';
    for (sz = 0; mi != m; ++mi)
    {
      if (*di == *m->data_.dirname && (m->data_.flags & DQMNet::DQM_PROP_TAGGED))
      {
        sprintf(tagbuf, "/%u", mi->data_.tag);
        if (sz > 0)
          *istr += ',';
        *istr += m->data_.objname;
        *istr += tagbuf;
        ++sz;
      }
    }
  }
}

std::vector<MonitorElement*>
DQMStore::getAllContents(const std::string &path,
                         uint32_t runNumber /* = 0 */,
                         uint32_t lumi /* = 0 */) const
{
  std::string clean;
  const std::string *cleaned = nullptr;
  cleanTrailingSlashes(path, clean, cleaned);
  MonitorElement proto(cleaned, std::string(), runNumber);
  proto.setLumi(lumi);

  std::vector<MonitorElement *> result;
  auto e = data_.end();
  auto i = data_.lower_bound(proto);
  for ( ; i != e && isSubdirectory(*cleaned, *i->data_.dirname); ++i) {
    if (runNumber != 0) {
      if (i->data_.run > runNumber // TODO[rovere]: pleonastic? first we encounter local ME of the same run ...
          || i->data_.streamId != 0
          || i->data_.moduleId != 0)
        break;
    }
    if (lumi != 0) {
      if (i->data_.lumi > lumi
          || i->data_.streamId != 0
          || i->data_.moduleId != 0)
        break;
    }
    if (runNumber != 0 or lumi !=0) {
      assert(i->data_.streamId == 0);
      assert(i->data_.moduleId == 0);
    }
    result.push_back(const_cast<MonitorElement *>(&*i));
  }

  if (enableMultiThread_)
    {
      //save legacy modules when running MT
      i = data_.begin();
      for ( ; i != e && isSubdirectory(*cleaned, *i->data_.dirname); ++i) {
        if (i->data_.run != 0 || i->data_.streamId != 0 || i->data_.moduleId != 0) break;
        result.push_back(const_cast<MonitorElement *>(&*i));
      }
    }

  return result;
}

std::vector<MonitorElement*>
DQMStore::getMatchingContents(const std::string &pattern, lat::Regexp::Syntax syntaxType /* = Wildcard */) const
{
  lat::Regexp rx;
  try
  {
    rx = lat::Regexp(pattern, 0, syntaxType);
    rx.study();
  }
  catch (lat::Error &e)
  {
    raiseDQMError("DQMStore", "Invalid regular expression '%s': %s",
                  pattern.c_str(), e.explain().c_str());
  }

  std::string path;
  std::vector<MonitorElement *> result;
  auto i = data_.begin();
  auto e = data_.end();
  for ( ; i != e; ++i)
  {
    path.clear();
    mergePath(path, *i->data_.dirname, i->data_.objname);
    if (rx.match(path))
      result.push_back(const_cast<MonitorElement *>(&*i));
  }

  return result;
}


void
DQMStore::reset()
{
  auto mi = data_.begin();
  auto me = data_.end();
  for ( ; mi != me; ++mi)
  {
    auto &me = const_cast<MonitorElement &>(*mi);
    if (mi->wasUpdated())
    {
      if (me.resetMe())
        me.Reset();
      me.resetUpdate();
    }
  }

  reset_ = true;
}


void
DQMStore::forceReset()
{
  auto mi = data_.begin();
  auto me = data_.end();
  for ( ; mi != me; ++mi)
  {
    if (forceResetOnBeginLumi_ && ((*mi).getLumiFlag() == false))
      continue;
    auto &me = const_cast<MonitorElement &>(*mi);
    me.Reset();
    me.resetUpdate();
  }

  reset_ = true;
}



void
DQMStore::postGlobalBeginLumi(const edm::GlobalContext &gc)
{
  static const std::string null_str("");

  auto const& lumiblock = gc.luminosityBlockID();
  uint32_t run = lumiblock.run();

  // find the range of non-legacy global MEs for the current run:
  // run != 0, lumi == 0 (implicit), stream id == 0, module id == 0
  const MonitorElement begin(&null_str, null_str, run, 0, 0);
  const MonitorElement end(&null_str, null_str, run, 0, 1);
  auto i = data_.lower_bound(begin);
  const auto e = data_.lower_bound(end);
  while (i != e) {
    auto& me = const_cast<MonitorElement&>(*i++);
    // skip per-run MEs
    if (not LSbasedMode_ and not me.getLumiFlag())
      continue;
    me.Reset();
    me.resetUpdate();
  }
}


void
DQMStore::deleteUnusedLumiHistograms(uint32_t run, uint32_t lumi)
{
  if (!enableMultiThread_)
    return;

  std::lock_guard<std::mutex> guard(book_mutex_);

  std::string null_str("");
  MonitorElement proto(&null_str, null_str, run, 0, 0);
  proto.setLumi(lumi);

  auto e = data_.end();
  auto i = data_.lower_bound(proto);

  while (i != e) {
    if (i->data_.streamId != 0 ||
        i->data_.moduleId != 0)
      break;
    if (i->data_.lumi != lumi)
      break;
    if (i->data_.run != run)
      break;
    if (not i->markedToDelete()) {
      ++i;
      continue;
    }

    if (verbose_ > 1) {
      std::cout << "DQMStore::deleteUnusedLumiHistograms: deleted monitor element '"
                << *i->data_.dirname << "/" << i->data_.objname << "'"
                << "flags " << i->data_.flags << "\n";
    }

    i = data_.erase(i);
  }
}

bool
DQMStore::extract(TObject *obj, const std::string &dir,
  bool overwrite, bool collateHistograms)
{
  // NB: Profile histograms inherit from TH*D, checking order matters.
  MonitorElement *refcheck = nullptr;
  if (auto *h = dynamic_cast<TProfile *>(obj))
  {
    MonitorElement *me = findObject(dir, h->GetName());
    if (! me)
      me = bookProfile_(dir, h->GetName(), (TProfile *) h->Clone());
    else if (overwrite)
      me->copyFrom(h);
    else if (isCollateME(me) || collateHistograms)
      collateProfile(me, h, verbose_);
    refcheck = me;
  }
  else if (auto *h = dynamic_cast<TProfile2D *>(obj))
  {
    MonitorElement *me = findObject(dir, h->GetName());
    if (! me)
      me = bookProfile2D_(dir, h->GetName(), (TProfile2D *) h->Clone());
    else if (overwrite)
      me->copyFrom(h);
    else if (isCollateME(me) || collateHistograms)
      collateProfile2D(me, h, verbose_);
    refcheck = me;
  }
  else if (auto *h = dynamic_cast<TH1F *>(obj))
  {
    MonitorElement *me = findObject(dir, h->GetName());
    if (! me)
      me = book1D_(dir, h->GetName(), (TH1F *) h->Clone());
    else if (overwrite)
      me->copyFrom(h);
    else if (isCollateME(me) || collateHistograms)
      collate1D(me, h, verbose_);
    refcheck = me;
  }
  else if (auto *h = dynamic_cast<TH1S *>(obj))
  {
    MonitorElement *me = findObject(dir, h->GetName());
    if (! me)
      me = book1S_(dir, h->GetName(), (TH1S *) h->Clone());
    else if (overwrite)
      me->copyFrom(h);
    else if (isCollateME(me) || collateHistograms)
      collate1S(me, h, verbose_);
    refcheck = me;
  }
  else if (auto *h = dynamic_cast<TH1D *>(obj))
  {
    MonitorElement *me = findObject(dir, h->GetName());
    if (! me)
      me = book1DD_(dir, h->GetName(), (TH1D *) h->Clone());
    else if (overwrite)
      me->copyFrom(h);
    else if (isCollateME(me) || collateHistograms)
      collate1DD(me, h, verbose_);
    refcheck = me;
  }
  else if (auto *h = dynamic_cast<TH2F *>(obj))
  {
    MonitorElement *me = findObject(dir, h->GetName());
    if (! me)
      me = book2D_(dir, h->GetName(), (TH2F *) h->Clone());
    else if (overwrite)
      me->copyFrom(h);
    else if (isCollateME(me) || collateHistograms)
      collate2D(me, h, verbose_);
    refcheck = me;
  }
  else if (auto *h = dynamic_cast<TH2S *>(obj))
  {
    MonitorElement *me = findObject(dir, h->GetName());
    if (! me)
      me = book2S_(dir, h->GetName(), (TH2S *) h->Clone());
    else if (overwrite)
      me->copyFrom(h);
    else if (isCollateME(me) || collateHistograms)
      collate2S(me, h, verbose_);
    refcheck = me;
  }
  else if (auto *h = dynamic_cast<TH2D *>(obj))
  {
    MonitorElement *me = findObject(dir, h->GetName());
    if (! me)
      me = book2DD_(dir, h->GetName(), (TH2D *) h->Clone());
    else if (overwrite)
      me->copyFrom(h);
    else if (isCollateME(me) || collateHistograms)
      collate2DD(me, h, verbose_);
    refcheck = me;
  }
  else if (auto *h = dynamic_cast<TH3F *>(obj))
  {
    MonitorElement *me = findObject(dir, h->GetName());
    if (! me)
      me = book3D_(dir, h->GetName(), (TH3F *) h->Clone());
    else if (overwrite)
      me->copyFrom(h);
    else if (isCollateME(me) || collateHistograms)
      collate3D(me, h, verbose_);
    refcheck = me;
  }
  else if (dynamic_cast<TObjString *>(obj))
  {
    lat::RegexpMatch m;
    if (! s_rxmeval.match(obj->GetName(), 0, 0, &m))
    {
      if (strstr(obj->GetName(), "CMSSW"))
      {
        if (verbose_)
          std::cout << "Input file version: " << obj->GetName() << std::endl;
        return true;
      }
      else if (strstr(obj->GetName(), "DQMPATCH"))
      {
        if (verbose_)
          std::cout << "DQM patch version: " << obj->GetName() << std::endl;
        return true;
      }
      else
      {
        std::cout << "*** DQMStore: WARNING: cannot extract object '"
                  << obj->GetName() << "' of type '"
                  << obj->IsA()->GetName() << "'\n";
        return false;
      }
    }

    std::string label = m.matchString(obj->GetName(), 1);
    std::string kind = m.matchString(obj->GetName(), 2);
    std::string value = m.matchString(obj->GetName(), 3);

    if (kind == "i")
    {
      MonitorElement *me = findObject(dir, label);
      if (! me || overwrite)
      {
        if (! me) me = bookInt_(dir, label);
        me->Fill(atoll(value.c_str()));
      }
    }
    else if (kind == "f")
    {
      MonitorElement *me = findObject(dir, label);
      if (! me || overwrite)
      {
        if (! me) me = bookFloat_(dir, label);
        me->Fill(atof(value.c_str()));
      }
    }
    else if (kind == "s")
    {
      MonitorElement *me = findObject(dir, label);
      if (! me)
        me = bookString_(dir, label, value);
      else if (overwrite)
        me->Fill(value);
    }
    else if (kind == "e")
    {
      MonitorElement *me = findObject(dir, label);
      if (! me)
      {
        std::cout << "*** DQMStore: WARNING: no monitor element '"
                  << label << "' in directory '"
                  << dir << "' to be marked as efficiency plot.\n";
        return false;
      }
      me->setEfficiencyFlag();
    }
    else if (kind == "t")
    {
      MonitorElement *me = findObject(dir, label);
      if (! me)
      {
        std::cout << "*** DQMStore: WARNING: no monitor element '"
                  << label << "' in directory '"
                  << dir << "' for a tag\n";
        return false;
      }
      errno = 0;
      char *endp = nullptr;
      unsigned long val = strtoul(value.c_str(), &endp, 10);
      if ((val == 0 && errno) || *endp || val > ~uint32_t(0))
      {
        std::cout << "*** DQMStore: WARNING: cannot restore tag '"
                  << value << "' for monitor element '"
                  << label << "' in directory '"
                  << dir << "' - invalid value\n";
        return false;
      }
      tag(me, val);
    }
    else if (kind == "qr")
    {
      // Handle qreports, but skip them while reading in references.
      if (! isSubdirectory(s_referenceDirName, dir))
      {
        size_t dot = label.find('.');
        if (dot == std::string::npos)
        {
          std::cout << "*** DQMStore: WARNING: quality report label in '" << label
                    << "' is missing a '.' and cannot be extracted\n";
          return false;
        }

        std::string mename (label, 0, dot);
        std::string qrname (label, dot+1, std::string::npos);

        m.reset();
        DQMNet::QValue qv;
        if (s_rxmeqr1.match(value, 0, 0, &m))
        {
          qv.code = atoi(m.matchString(value, 1).c_str());
          qv.qtresult = strtod(m.matchString(value, 2).c_str(), nullptr);
          qv.message = m.matchString(value, 4);
          qv.qtname = qrname;
          qv.algorithm = m.matchString(value, 3);
        }
        else if (s_rxmeqr2.match(value, 0, 0, &m))
        {
          qv.code = atoi(m.matchString(value, 1).c_str());
          qv.qtresult = 0; // unavailable in old format
          qv.message = m.matchString(value, 2);
          qv.qtname = qrname;
          // qv.algorithm unavailable in old format
        }
        else
        {
          std::cout << "*** DQMStore: WARNING: quality test value '"
                    << value << "' is incorrectly formatted\n";
          return false;
        }

        MonitorElement *me = findObject(dir, mename);
        if (! me)
        {
          std::cout << "*** DQMStore: WARNING: no monitor element '"
                    << mename << "' in directory '"
                    << dir << "' for quality test '"
                    << label << "'\n";
          return false;
        }

        me->addQReport(qv, /* FIXME: getQTest(qv.qtname)? */ nullptr);
      }
    }
    else
    {
      std::cout << "*** DQMStore: WARNING: cannot extract object '"
                << obj->GetName() << "' of type '"
                << obj->IsA()->GetName() << "'\n";
      return false;
    }
  }
  else if (auto *n = dynamic_cast<TNamed *>(obj))
  {
    // For old DQM data.
    std::string s;
    s.reserve(6 + strlen(n->GetTitle()) + 2*strlen(n->GetName()));
    s += '<'; s += n->GetName(); s += '>';
    s += n->GetTitle();
    s += '<'; s += '/'; s += n->GetName(); s += '>';
    TObjString os(s.c_str());
    return extract(&os, dir, overwrite, collateHistograms_);
  }
  else
  {
    std::cout << "*** DQMStore: WARNING: cannot extract object '"
              << obj->GetName() << "' of type '" << obj->IsA()->GetName()
              << "' and with title '" << obj->GetTitle() << "'\n";
    return false;
  }

  // If we just read in a reference MonitorElement, and there is a
  // MonitorElement with the same name, link the two together.
  // The other direction is handled by the book() method.
  if (refcheck && isSubdirectory(s_referenceDirName, dir))
  {
    std::string mdir(dir, s_referenceDirName.size()+1, std::string::npos);
    if (MonitorElement *master = findObject(mdir, obj->GetName()))
    {
      // We have extracted a MonitorElement, and it's located in the reference
      // dir. Then we find the corresponding MonitorElement in the
      // non-reference dir and assign the object_ of the reference
      // MonitorElement to the reference_ property of the corresponding
      // non-reference MonitorElement.
      master->data_.flags |= DQMNet::DQM_PROP_HAS_REFERENCE;
      master->reference_ = refcheck->object_;
    }
  }

  return true;
}

bool
DQMStore::cdInto(const std::string &path) const
{
  assert(! path.empty());

  // Find the first path component.
  size_t start = 0;
  size_t end = path.find('/', start);
  if (end == std::string::npos)
    end = path.size();

  while (true)
  {
    // Check if this subdirectory component exists.  If yes, make sure
    // it is actually a subdirectory.  Otherwise create or cd into it.
    std::string part(path, start, end-start);
    TObject *o = gDirectory->Get(part.c_str());
    if (o && ! dynamic_cast<TDirectory *>(o))
      raiseDQMError("DQMStore", "Attempt to create directory '%s' in a file"
                    " fails because the part '%s' already exists and is not"
                    " directory", path.c_str(), part.c_str());
    else if (! o)
      gDirectory->mkdir(part.c_str());

    if (! gDirectory->cd(part.c_str()))
      raiseDQMError("DQMStore", "Attempt to create directory '%s' in a file"
                    " fails because could not cd into subdirectory '%s'",
                    path.c_str(), part.c_str());

    // Stop if we reached the end, ignoring any trailing '/'.
    if (end+1 >= path.size())
      break;

    // Find the next path component.
    start = end+1;
    end = path.find('/', start);
    if (end == std::string::npos)
      end = path.size();
  }

  return true;
}

void
DQMStore::saveMonitorElementToROOT(
    MonitorElement const& me,
    TFile & file)
{
  // Save the object.
  if (me.kind() < MonitorElement::DQM_KIND_TH1F) {
    TObjString(me.tagString().c_str()).Write();
  } else {
    me.object_->Write();
  }

  // Save quality reports if this is not in reference section.
  if (not isSubdirectory(s_referenceDirName, *me.data_.dirname))
  {
    for (auto const& report: me.data_.qreports) {
      TObjString(me.qualityTagString(report).c_str()).Write();
    }
  }

  // Save efficiency tag, if any.
  if (me.data_.flags & DQMNet::DQM_PROP_EFFICIENCY_PLOT) {
    TObjString(me.effLabelString().c_str()).Write();
  }

  // Save tag if any.
  if (me.data_.flags & DQMNet::DQM_PROP_TAGGED) {
    TObjString(me.tagLabelString().c_str()).Write();
  }
}

void
DQMStore::saveMonitorElementRangeToROOT(
    std::string const& dir,
    std::string const& refpath,
    SaveReferenceTag ref,
    int minStatus,
    unsigned int run,
    MEMap::const_iterator begin,
    MEMap::const_iterator end,
    TFile & file,
    unsigned int & counter)
{
  for (auto const& me: boost::make_iterator_range(begin, end))
  {
    if (not isSubdirectory(dir, *me.data_.dirname))
      break;

    if (verbose_ > 1)
      std::cout << "DQMStore::save:"
                << " run: " << me.run()
                << " lumi: " << me.lumi()
                << " lumiFlag: " << me.getLumiFlag()
                << " streamId: " << me.streamId()
                << " moduleId: " << me.moduleId()
                << " fullpathname: " << me.getFullname()
                << " flags: " << std::hex << me.data_.flags
                << std::endl;

    // Skip MonitorElements in a subdirectory of the current one.
    if (dir != *me.data_.dirname) {
      if (verbose_ > 1) {
        std::cout << "DQMStore::save: skipping monitor element in a subfolder of " << dir << "/" << std::endl;
      }
      continue;
    }

    // For MonitorElements booked with the thread-safe approach, identified
    // by having run != 0, ignore the per-stream ones.
    if (run != 0 and (me.data_.streamId != 0 or me.data_.moduleId != 0)) {
      if (verbose_ > 1) {
        std::cout << "DQMStore::save: skipping per-stream monitor element" << std::endl;
      }
      continue;
    }

    // Handle reference histograms, with three distinct cases:
    // 1) Skip all references entirely on saving.
    // 2) Blanket saving of all references.
    // 3) Save only references for monitor elements with qtests.
    // The latter two are affected by "path" sub-tree selection,
    // i.e. references are saved only in the selected tree part.
    if (isSubdirectory(refpath, *me.data_.dirname))
    {
      if (ref == SaveWithoutReference)
        // Skip the reference entirely.
        continue;
      else if (ref == SaveWithReference)
        // Save all references regardless of qtests.
        ;
      else if (ref == SaveWithReferenceForQTest)
      {
        // Save only references for monitor elements with qtests
        // with an optional cut on minimum quality test result.
        int status = -1;
        std::string mname(me.getFullname(), s_referenceDirName.size()+1, std::string::npos);
        MonitorElement *master = get(mname);
        if (master)
          for (size_t i = 0, e = master->data_.qreports.size(); i != e; ++i)
            status = std::max(status, master->data_.qreports[i].code);

        if (not master or status < minStatus)
        {
          if (verbose_ > 1)
            std::cout << "DQMStore::save: skipping monitor element '"
                      << me.data_.objname << "' while saving, status is "
                      << status << ", required minimum status is "
                      << minStatus << std::endl;
          continue;
        }
      }
    }

    if (verbose_ > 1) {
      std::cout << "DQMStore::save: saving monitor element" << std::endl;
    }

    saveMonitorElementToROOT(me, file);

    // Count saved histograms
    ++counter;
  }
}

void
DQMStore::save(const std::string &filename,
               const std::string &path /* = "" */,
               const std::string &pattern /* = "" */,
               const std::string &rewrite /* = "" */,
               const uint32_t run /* = 0 */,
               const uint32_t lumi /* = 0 */,
               SaveReferenceTag ref /* = SaveWithReference */,
               int minStatus /* = dqm::qstatus::STATUS_OK */,
               const std::string &fileupdate /* = RECREATE */)
{
  // TFile flushes to disk with fsync() on every TDirectory written to
  // the file.  This makes DQM file saving painfully slow, and
  // ironically makes it _more_ likely the file saving gets
  // interrupted and corrupts the file.  The utility class below
  // simply ignores the flush synchronisation.
  class TFileNoSync : public TFile
  {
  public:
    TFileNoSync(const char *file, const char *opt) : TFile(file, opt) {}
    Int_t SysSync(Int_t) override { return 0; }
  };

  std::lock_guard<std::mutex> guard(book_mutex_);

  unsigned int nme = 0;

  // open output file, on 1st save recreate, later update
  if (verbose_) {
    std::cout << "DQMStore::save: Opening TFile '" << filename
              << "' with option '" << fileupdate << "'"
              << std::endl;
  }

  TFileNoSync f(filename.c_str(), fileupdate.c_str()); // open file
  if(f.IsZombie())
    raiseDQMError("DQMStore", "Failed to create/update file '%s'", filename.c_str());
  f.cd();

  // Construct a regular expression from the pattern string.
  std::unique_ptr<lat::Regexp> rxpat;
  if (not pattern.empty())
    rxpat = std::make_unique<lat::Regexp>(pattern);

  // Prepare a path for the reference object selection.
  std::string refpath;
  refpath.reserve(s_referenceDirName.size() + path.size() + 2);
  refpath += s_referenceDirName;
  if (not path.empty())
  {
    refpath += '/';
    refpath += path;
  }

  // Loop over the directory structure.
  for (auto const& dir: dirs_)
  {
    // Check if we should process this directory.  We process the
    // requested part of the object tree, including references.
    if (not path.empty()
        and not isSubdirectory(refpath, dir)
        and not isSubdirectory(path, dir))
      continue;

    if (verbose_ > 1) {
      std::cout << "DQMStore::save: DQM folder " << dir << "/" << std::endl;
    }

    // Create the directory.
    gDirectory->cd("/");
    if (dir.empty())
      cdInto(s_monitorDirName);
    else if (rxpat.get())
      cdInto(s_monitorDirName + '/' + lat::StringOps::replace(dir, *rxpat, rewrite));
    else
      cdInto(s_monitorDirName + '/' + dir);

    // Loop over monitor elements in this directory.
    if (not enableMultiThread_) {
      MonitorElement proto(&dir, std::string(), run, 0, 0);
      auto begin = data_.lower_bound(proto);
      auto end   = data_.end();
      saveMonitorElementRangeToROOT(dir, refpath, ref, minStatus, run, begin, end, f, nme);
    } else {
      // Restrict the loop to the monitor elements for the current lumisection
      MonitorElement proto(&dir, std::string(), run, 0, 0);
      proto.setLumi(lumi);
      auto begin = data_.lower_bound(proto);
      proto.setLumi(lumi+1);
      auto end   = data_.lower_bound(proto);
      saveMonitorElementRangeToROOT(dir, refpath, ref, minStatus, run, begin, end, f, nme);
    }

    // In LSbasedMode, loop also over the (run, 0, 0, 0) global histograms;
    // these could be the merged global histrograms of their per-stream
    // counterparts after the streamEndRun transition - but they are not
    // produced in LSbasedMode.
    if (enableMultiThread_ and LSbasedMode_ and lumi != 0) {
      auto begin = data_.lower_bound(MonitorElement(&dir, std::string(), run, 0, 0));
      auto end   = data_.lower_bound(MonitorElement(&dir, std::string(), run, 0, 1));
      saveMonitorElementRangeToROOT(dir, refpath, ref, minStatus, run, begin, end, f, nme);
    }
  }

  f.Close();

  // Maybe make some noise.
  if (verbose_) {
    std::cout << "DQMStore::save: successfully wrote " << nme
              << " objects from path '" << path << "/"
              << "' into DQM file '" << filename << "'\n";
  }
}

void
DQMStore::saveMonitorElementToPB(
    MonitorElement const& me,
    dqmstorepb::ROOTFilePB & file)
{
  // Save the object.
  TBufferFile buffer(TBufferFile::kWrite);
  if (me.kind() < MonitorElement::DQM_KIND_TH1F) {
    TObjString object(me.tagString().c_str());
    buffer.WriteObject(&object);
  } else {
    buffer.WriteObject(me.object_);
  }
  dqmstorepb::ROOTFilePB::Histo & histo = * file.add_histo();
  histo.set_full_pathname(*me.data_.dirname + '/' + me.data_.objname);
  histo.set_flags(me.data_.flags);
  histo.set_size(buffer.Length());
  histo.set_streamed_histo((const void*)buffer.Buffer(), buffer.Length());

  // Save quality reports if this is not in reference section.
  // XXX not supported by protobuf files.

  // Save efficiency tag, if any.
  // XXX not supported by protobuf files.

  // Save tag if any.
  // XXX not supported by protobuf files.
}

void
DQMStore::saveMonitorElementRangeToPB(
    std::string const& dir,
    unsigned int run,
    MEMap::const_iterator begin,
    MEMap::const_iterator end,
    dqmstorepb::ROOTFilePB & file,
    unsigned int & counter)
{
  for (auto const& me: boost::make_iterator_range(begin, end))
  {
    if (not isSubdirectory(dir, *me.data_.dirname))
      break;

    if (verbose_ > 1)
      std::cout << "DQMStore::savePB:"
                << " run: " << me.run()
                << " lumi: " << me.lumi()
                << " lumiFlag: " << me.getLumiFlag()
                << " streamId: " << me.streamId()
                << " moduleId: " << me.moduleId()
                << " fullpathname: " << me.getFullname()
                << " flags: " << std::hex << me.data_.flags
                << std::endl;

    // Skip MonitorElements in a subdirectory of the current one.
    if (dir != *me.data_.dirname) {
      if (verbose_ > 1) {
        std::cout << "DQMStore::savePB: skipping monitor element in a subfolder of " << dir << "/" << std::endl;
      }
      continue;
    }

    // For MonitorElements booked with the thread-safe approach, identified
    // by having run != 0, ignore the per-stream ones.
    if (run != 0 and (me.data_.streamId != 0 or me.data_.moduleId != 0)) {
      if (verbose_ > 1) {
        std::cout << "DQMStore::savePB: skipping per-stream monitor element" << std::endl;
      }
      continue;
    }

    // Handle reference histograms, with three distinct cases:
    // XXX not supported by protobuf files.

    if (verbose_ > 1) {
      std::cout << "DQMStore::savePB: saving monitor element" << std::endl;
    }

    saveMonitorElementToPB(me, file);

    // Count saved histograms
    ++counter;
  }
}

void
DQMStore::savePB(const std::string &filename,
                 const std::string &path /* = "" */,
                 const uint32_t run /* = 0 */,
                 const uint32_t lumi /* = 0 */)
{
  using google::protobuf::io::FileOutputStream;
  using google::protobuf::io::GzipOutputStream;
  using google::protobuf::io::StringOutputStream;

  std::lock_guard<std::mutex> guard(book_mutex_);

  unsigned int nme = 0;

  if (verbose_) {
    std::cout << "DQMStore::savePB: Opening PBFile '" << filename << "'"
              << std::endl;
  }
  dqmstorepb::ROOTFilePB dqmstore_message;

  // Loop over the directory structure.
  for (auto const& dir: dirs_)
  {
    // Check if we should process this directory.  We process the
    // requested part of the object tree, including references.
    if (not path.empty()
        and not isSubdirectory(path, dir))
      continue;

    if (verbose_ > 1) {
      std::cout << "DQMStore::savePB: DQM folder " << dir << "/" << std::endl;
    }

    // Loop over monitor elements in this directory.
    if (not enableMultiThread_) {
      MonitorElement proto(&dir, std::string(), run, 0, 0);
      auto begin = data_.lower_bound(proto);
      auto end   = data_.end();
      saveMonitorElementRangeToPB(dir, run, begin, end, dqmstore_message, nme);
    } else {
      // Restrict the loop to the monitor elements for the current lumisection
      MonitorElement proto(&dir, std::string(), run, 0, 0);
      proto.setLumi(lumi);
      auto begin = data_.lower_bound(proto);
      proto.setLumi(lumi+1);
      auto end   = data_.lower_bound(proto);
      saveMonitorElementRangeToPB(dir, run, begin, end, dqmstore_message, nme);
    }

    // In LSbasedMode, loop also over the (run, 0, 0, 0) global histograms;
    // these could be the merged global histrograms of their per-stream
    // counterparts after the streamEndRun transition - but they are not
    // produced in LSbasedMode.
    if (enableMultiThread_ and LSbasedMode_ and lumi != 0) {
      auto begin = data_.lower_bound(MonitorElement(&dir, std::string(), run, 0, 0));
      auto end   = data_.lower_bound(MonitorElement(&dir, std::string(), run, 0, 1));
      saveMonitorElementRangeToPB(dir, run, begin, end, dqmstore_message, nme);
    }
  }

  int filedescriptor = ::open(filename.c_str(),
                              O_WRONLY | O_CREAT | O_TRUNC,
                              S_IRUSR | S_IWUSR |
                              S_IRGRP | S_IWGRP |
                              S_IROTH);
  FileOutputStream file_stream(filedescriptor);
  GzipOutputStream::Options options;
  options.format = GzipOutputStream::GZIP;
  options.compression_level = 1;
  GzipOutputStream gzip_stream(&file_stream, options);
  dqmstore_message.SerializeToZeroCopyStream(&gzip_stream);

  // Flush the internal streams before closing the fd.
  gzip_stream.Close();
  file_stream.Close();
  ::close(filedescriptor);

  // Maybe make some noise.
  if (verbose_) {
    std::cout << "DQMStore::savePB: successfully wrote " << nme
              << " objects from path '" << path << "/"
              << "' into DQM file '" << filename << "'\n";
  }
}


unsigned int
DQMStore::readDirectory(TFile *file,
                        bool overwrite,
                        const std::string &onlypath,
                        const std::string &prepend,
                        const std::string &curdir,
                        OpenRunDirs stripdirs)
{
  unsigned int ntot = 0;
  unsigned int count = 0;

  if (! file->cd(curdir.c_str()))
    raiseDQMError("DQMStore", "Failed to process directory '%s' while"
                  " reading file '%s'", curdir.c_str(), file->GetName());

  // Figure out current directory name, but strip out the top
  // directory into which we dump everything.
  std::string dirpart = curdir;
  if (dirpart.compare(0, s_monitorDirName.size(), s_monitorDirName) == 0)
  {
    if (dirpart.size() == s_monitorDirName.size())
      dirpart.clear();
    else if (dirpart[s_monitorDirName.size()] == '/')
      dirpart.erase(0, s_monitorDirName.size()+1);
  }

  // See if we are going to skip this directory.
  bool skip = (! onlypath.empty() && ! isSubdirectory(onlypath, dirpart));

  if (prepend == s_collateDirName ||
      prepend == s_referenceDirName ||
      stripdirs == StripRunDirs )
  {
    // Remove Run # and RunSummary dirs
    // first look for Run summary,
    // if that is found and erased, also erase Run dir
    size_t slash = dirpart.find('/');
    size_t pos = dirpart.find("/Run summary");
    if (slash != std::string::npos && pos !=std::string::npos)
    {
      dirpart.erase(pos,12);

      pos = dirpart.find("Run ");
      size_t length = dirpart.find('/',pos+1)-pos+1;
      if (pos !=std::string::npos)
        dirpart.erase(pos,length);
    }
  }

  // If we are prepending, add it to the directory name,
  // and suppress reading of already existing reference histograms
  if (prepend == s_collateDirName ||
      prepend == s_referenceDirName)
  {
    size_t slash = dirpart.find('/');
    // If we are reading reference, skip previous reference.
    if (slash == std::string::npos   // skip if Reference is toplevel folder, i.e. no slash
        && slash+1+s_referenceDirName.size() == dirpart.size()
        && dirpart.compare(slash+1, s_referenceDirName.size(), s_referenceDirName) == 0)
      return 0;

    slash = dirpart.find('/');
    // Skip reading of EventInfo subdirectory.
    if (slash != std::string::npos
        && slash + 10 == dirpart.size()
        && dirpart.compare( slash+1 , 9 , "EventInfo") == 0) {
      if (verbose_)
        std::cout << "DQMStore::readDirectory: skipping '" << dirpart << "'\n";
      return 0;
    }

    // Add prefix.
    if (dirpart.empty())
      dirpart = prepend;
    else
      dirpart = prepend + '/' + dirpart;
  }
  else if (! prepend.empty())
  {
    if (dirpart.empty())
      dirpart = prepend;
    else
      dirpart = prepend + '/' + dirpart;
  }

  // Loop over the contents of this directory in the file.
  // Post-pone string object handling to happen after other
  // objects have been read in so we are guaranteed to have
  // histograms by the time we read in quality tests and tags.
  TKey *key;
  TIter next (gDirectory->GetListOfKeys());
  std::list<TObject *> delayed;
  while ((key = (TKey *) next()))
  {
    std::unique_ptr<TObject> obj(key->ReadObj());
    if (dynamic_cast<TDirectory *>(obj.get()))
    {
      std::string subdir;
      subdir.reserve(curdir.size() + strlen(obj->GetName()) + 2);
      subdir += curdir;
      if (! curdir.empty())
        subdir += '/';
      subdir += obj->GetName();

      ntot += readDirectory(file, overwrite, onlypath, prepend, subdir, stripdirs);
    }
    else if (skip)
      ;
    else if (dynamic_cast<TObjString *>(obj.get()))
    {
      delayed.push_back(obj.release());
    }
    else
    {
      if (verbose_ > 2)
        std::cout << "DQMStore: reading object '" << obj->GetName()
                  << "' of type '" << obj->IsA()->GetName()
                  << "' from '" << file->GetName()
                  << "' into '" << dirpart << "'\n";

      makeDirectory(dirpart);
      if (extract(obj.get(), dirpart, overwrite, collateHistograms_))
        ++count;
    }
  }

  while (! delayed.empty())
  {
    if (verbose_ > 2)
      std::cout << "DQMStore: reading object '" << delayed.front()->GetName()
                << "' of type '" << delayed.front()->IsA()->GetName()
                << "' from '" << file->GetName()
                << "' into '" << dirpart << "'\n";

    makeDirectory(dirpart);
    if (extract(delayed.front(), dirpart, overwrite, collateHistograms_))
      ++count;

    delete delayed.front();
    delayed.pop_front();
  }

  if (verbose_ > 1)
    std::cout << "DQMStore: read " << count << '/' << ntot
              << " objects from directory '" << dirpart << "'\n";

  return ntot + count;
}

bool
DQMStore::open(const std::string &filename,
               bool overwrite /* = false */,
               const std::string &onlypath /* ="" */,
               const std::string &prepend /* ="" */,
               OpenRunDirs stripdirs /* =KeepRunDirs */,
               bool fileMustExist /* =true */)
{
  return readFile(filename,overwrite,onlypath,prepend,stripdirs,fileMustExist);
}

bool
DQMStore::load(const std::string &filename,
               OpenRunDirs stripdirs /* =StripRunDirs */,
               bool fileMustExist /* =true */)
{
  bool overwrite = true;
  if (collateHistograms_) overwrite = false;
  if (verbose_)
  {
    std::cout << "DQMStore::load: reading from file '" << filename << "'\n";
    if (collateHistograms_)
      std::cout << "DQMStore::load: in collate mode   " << "\n";
    else
      std::cout << "DQMStore::load: in overwrite mode   " << "\n";
  }

  if (!s_rxpbfile.match(filename, 0, 0))
    return readFile(filename, overwrite, "", "", stripdirs, fileMustExist);
  else
    return readFilePB(filename, overwrite, "", "", stripdirs, fileMustExist);
}

bool
DQMStore::readFile(const std::string &filename,
                   bool overwrite /* = false */,
                   const std::string &onlypath /* ="" */,
                   const std::string &prepend /* ="" */,
                   OpenRunDirs stripdirs /* =StripRunDirs */,
                   bool fileMustExist /* =true */)
{

  if (verbose_)
    std::cout << "DQMStore::readFile: reading from file '" << filename << "'\n";

  std::unique_ptr<TFile> f;

  try
  {
    f.reset(TFile::Open(filename.c_str()));
    if (! f.get() || f->IsZombie())
      raiseDQMError("DQMStore", "Failed to open file '%s'", filename.c_str());
  }
  catch (std::exception &)
  {
    if (fileMustExist)
      throw;
    else
    {
    if (verbose_)
      std::cout << "DQMStore::readFile: file '" << filename << "' does not exist, continuing\n";
    return false;
    }
  }

  unsigned n = readDirectory(f.get(), overwrite, onlypath, prepend, "", stripdirs);
  f->Close();

  auto mi = data_.begin();
  auto me = data_.end();
  for ( ; mi != me; ++mi)
    const_cast<MonitorElement &>(*mi).updateQReportStats();

  if (verbose_)
  {
    std::cout << "DQMStore::open: successfully read " << n
              << " objects from file '" << filename << "'";
    if (! onlypath.empty())
      std::cout << " from directory '" << onlypath << "'";
    if (! prepend.empty())
      std::cout << " into directory '" << prepend << "'";
    std::cout << std::endl;
  }
  return true;
}

inline TObject * DQMStore::extractNextObject(TBufferFile &buf) const {
  if (buf.Length() == buf.BufferSize())
    return nullptr;
  buf.InitMap();
  void *ptr = buf.ReadObjectAny(nullptr);
  return reinterpret_cast<TObject *>(ptr);
}

void DQMStore::get_info(const dqmstorepb::ROOTFilePB::Histo &h,
                        std::string &dirname,
                        std::string &objname,
                        TObject ** obj) {

  size_t slash = h.full_pathname().rfind('/');
  size_t dirpos = (slash == std::string::npos ? 0 : slash);
  size_t namepos = (slash == std::string::npos ? 0 : slash+1);
  dirname.assign(h.full_pathname(), 0, dirpos);
  objname.assign(h.full_pathname(), namepos, std::string::npos);
  TBufferFile buf(TBufferFile::kRead, h.size(),
                  (void*)h.streamed_histo().data(),
                  kFALSE);
  buf.Reset();
  *obj = extractNextObject(buf);
  if (!*obj) {
    raiseDQMError("DQMStore", "Error reading element:'%s'" , h.full_pathname().c_str());
  }
}

bool
DQMStore::readFilePB(const std::string &filename,
             bool overwrite /* = false */,
             const std::string &onlypath /* ="" */,
             const std::string &prepend /* ="" */,
             OpenRunDirs stripdirs /* =StripRunDirs */,
             bool fileMustExist /* =true */)
{
  using google::protobuf::io::FileInputStream;
  using google::protobuf::io::FileOutputStream;
  using google::protobuf::io::GzipInputStream;
  using google::protobuf::io::GzipOutputStream;
  using google::protobuf::io::CodedInputStream;
  using google::protobuf::io::ArrayInputStream;

  if (verbose_)
    std::cout << "DQMStore::readFile: reading from file '" << filename << "'\n";

  int filedescriptor;
  if ((filedescriptor = ::open(filename.c_str(), O_RDONLY)) == -1) {
    if (fileMustExist)
      raiseDQMError("DQMStore", "Failed to open file '%s'", filename.c_str());
    else
      if (verbose_)
        std::cout << "DQMStore::readFile: file '" << filename << "' does not exist, continuing\n";
    return false;
  }

  dqmstorepb::ROOTFilePB dqmstore_message;
  FileInputStream fin(filedescriptor);
  GzipInputStream input(&fin);
  CodedInputStream input_coded(&input);
  input_coded.SetTotalBytesLimit(1024*1024*1024, -1);
  if (!dqmstore_message.ParseFromCodedStream(&input_coded)) {
    raiseDQMError("DQMStore", "Fatal parsing file '%s'", filename.c_str());
    return false;
  }
  ::close(filedescriptor);

  for (int i = 0; i < dqmstore_message.histo_size(); i++) {
    std::string path;
    std::string objname;

    TObject *obj = nullptr;
    const dqmstorepb::ROOTFilePB::Histo &h = dqmstore_message.histo(i);
    get_info(h, path, objname, &obj);

    setCurrentFolder(path);
    if (obj)
    {
      /* Before calling the extract() check if histogram exists:
       * if it does - flags for the given monitor are already set (and merged)
       * else - set the flags after the histogram is created.
       */
      MonitorElement *me = findObject(path, objname);

      /* Run histograms should be collated and not overwritten,
       * Lumi histograms should be overwritten (and collate flag is not checked)
       */
      bool overwrite = h.flags() & DQMNet::DQM_PROP_LUMI;
      bool collate = !(h.flags() & DQMNet::DQM_PROP_LUMI);
      extract(static_cast<TObject *>(obj), path, overwrite, collate);

      if (me == nullptr) {
        me = findObject(path, objname);
        me->data_.flags = h.flags();
      }

      delete obj;
    }
  }

  cd();
  return true;
}

void
DQMStore::rmdir(const std::string &path)
{
  std::string clean;
  const std::string *cleaned = nullptr;
  cleanTrailingSlashes(path, clean, cleaned);
  MonitorElement proto(cleaned, std::string());

  auto e = data_.end();
  auto i = data_.lower_bound(proto);
  while (i != e && isSubdirectory(*cleaned, *i->data_.dirname))
    data_.erase(i++);

  auto de = dirs_.end();
  auto di = dirs_.lower_bound(*cleaned);
  while (di != de && isSubdirectory(*cleaned, *di))
    dirs_.erase(di++);
}

void
DQMStore::removeContents(const std::string &dir)
{
  MonitorElement proto(&dir, std::string());
  auto e = data_.end();
  auto i = data_.lower_bound(proto);
  while (i != e && isSubdirectory(dir, *i->data_.dirname))
    if (dir == *i->data_.dirname)
      data_.erase(i++);
    else
      ++i;
}

void
DQMStore::removeContents()
{
  removeContents(pwd_);
}

void
DQMStore::removeElement(const std::string &name)
{
  removeElement(pwd_, name);
}

void
DQMStore::removeElement(const std::string &dir, const std::string &name, bool warning /* = true */)
{
  MonitorElement proto(&dir, name);
  auto pos = data_.find(proto);
  if (pos != data_.end())
    data_.erase(pos);
  else if (warning)
    std::cout << "DQMStore: WARNING: attempt to remove non-existent"
              << " monitor element '" << name << "' in '" << dir << "'\n";
}

QCriterion *
DQMStore::getQCriterion(const std::string &qtname) const
{
  auto i = qtests_.find(qtname);
  auto e = qtests_.end();
  return (i == e ? nullptr : i->second);
}

QCriterion *
DQMStore::createQTest(const std::string &algoname, const std::string &qtname)
{
  if (qtests_.count(qtname))
    raiseDQMError("DQMStore", "Attempt to create duplicate quality test '%s'",
                  qtname.c_str());

  auto i = qalgos_.find(algoname);
  if (i == qalgos_.end())
    raiseDQMError("DQMStore", "Cannot create a quality test using unknown"
                  " algorithm '%s'", algoname.c_str());

  QCriterion *qc = i->second(qtname);
  qc->setVerbose(verboseQT_);

  qtests_[qtname] = qc;
  return qc;
}

void
DQMStore::useQTest(const std::string &dir, const std::string &qtname)
{
  // Clean the path
  std::string clean;
  const std::string *cleaned = nullptr;
  cleanTrailingSlashes(dir, clean, cleaned);

  // Validate the path.
  if (cleaned->find_first_not_of(s_safe) != std::string::npos)
    raiseDQMError("DQMStore", "Monitor element path name '%s'"
                  " uses unacceptable characters", cleaned->c_str());

  // Redirect to the pattern match version.
  useQTestByMatch(*cleaned + "/*", qtname);
}

int
DQMStore::useQTestByMatch(const std::string &pattern, const std::string &qtname)
{
  QCriterion *qc = getQCriterion(qtname);
  if (! qc)
    raiseDQMError("DQMStore", "Cannot apply non-existent quality test '%s'",
                  qtname.c_str());

  auto * fm = new fastmatch( pattern );

  // Record the test for future reference.
  QTestSpec qts(fm, qc);
  qtestspecs_.push_back(qts);

  // Apply the quality test.
  auto mi = data_.begin();
  auto me = data_.end();
  std::string path;
  int cases = 0;
  for ( ; mi != me; ++mi)
  {
    path.clear();
    mergePath(path, *mi->data_.dirname, mi->data_.objname);
    if (fm->match(path))
    {
      ++cases;
      const_cast<MonitorElement &>(*mi).addQReport(qts.second);
    }
  }

  //return the number of matched cases
  return cases;
}
void
DQMStore::runQTests()
{

  if (verbose_ > 0)
    std::cout << "DQMStore: running runQTests() with reset = "
              << ( reset_ ? "true" : "false" ) << std::endl;

  // Apply quality tests to each monitor element, skipping references.
  auto mi = data_.begin();
  auto me = data_.end();
  for ( ; mi != me; ++mi)
    if (! isSubdirectory(s_referenceDirName, *mi->data_.dirname))
      const_cast<MonitorElement &>(*mi).runQTests();

  reset_ = false;
}

int
DQMStore::getStatus(const std::string &path /* = "" */) const
{
  std::string clean;
  const std::string *cleaned = nullptr;
  cleanTrailingSlashes(path, clean, cleaned);

  int status = dqm::qstatus::STATUS_OK;
  auto mi = data_.begin();
  auto me = data_.end();
  for ( ; mi != me; ++mi)
  {
    if (! cleaned->empty() && ! isSubdirectory(*cleaned, *mi->data_.dirname))
      continue;

    if (mi->hasError())
      return dqm::qstatus::ERROR;
    else if (mi->hasWarning())
      status = dqm::qstatus::WARNING;
    else if (status < dqm::qstatus::WARNING
             && mi->hasOtherReport())
      status = dqm::qstatus::OTHER;
  }
  return status;
}

void
DQMStore::softReset(MonitorElement *me)
{
  if (me)
    me->softReset();
}

// reverts action of softReset
void
DQMStore::disableSoftReset(MonitorElement *me)
{
  if (me)
    me->disableSoftReset();
}

void
DQMStore::setAccumulate(MonitorElement *me, bool flag)
{
  if (me)
    me->setAccumulate(flag);
}

void
DQMStore::showDirStructure() const
{
  std::vector<std::string> contents;
  getContents(contents);

  std::cout << " ------------------------------------------------------------\n"
            << "                    Directory structure:                     \n"
            << " ------------------------------------------------------------\n";

  std::copy(contents.begin(), contents.end(),
            std::ostream_iterator<std::string>(std::cout, "\n"));

  std::cout << " ------------------------------------------------------------\n";
}

// check if the collate option is active on the DQMStore
bool
DQMStore::isCollate() const
{
  return collateHistograms_;
}
// check if the monitor element is in auto-collation folder
bool
DQMStore::isCollateME(MonitorElement *me) const
{ return me && isSubdirectory(s_collateDirName, *me->data_.dirname); }

void
DQMStore::scaleElements()
{
  if (scaleFlag_ == 0.0) return;
  if (verbose_ > 0)
    std::cout << " =========== " << " ScaleFlag " << scaleFlag_ << std::endl;
  double factor = scaleFlag_;
  int events = 1;
  if (dirExists("Info/EventInfo")) {
    if ( scaleFlag_ == -1.0) {
      MonitorElement * scale_me = get("Info/EventInfo/ScaleFactor");
      if (scale_me && scale_me->kind()==MonitorElement::DQM_KIND_REAL) factor = scale_me->getFloatValue();
    }
    MonitorElement * event_me = get("Info/EventInfo/processedEvents");
    if (event_me && event_me->kind()==MonitorElement::DQM_KIND_INT) events = event_me->getIntValue();
  }
  factor = factor/(events*1.0);

  auto mi = data_.begin();
  auto me = data_.end();
  for ( ; mi != me; ++mi)
  {
    auto &me = const_cast<MonitorElement &>(*mi);
    switch (me.kind())
      {
      case MonitorElement::DQM_KIND_TH1F:
        {
          me.getTH1F()->Scale(factor);
          break;
        }
      case MonitorElement::DQM_KIND_TH1S:
        {
          me.getTH1S()->Scale(factor);
          break;
        }
      case MonitorElement::DQM_KIND_TH1D:
        {
          me.getTH1D()->Scale(factor);
          break;
        }
      case MonitorElement::DQM_KIND_TH2F:
        {
          me.getTH2F()->Scale(factor);
          break;
        }
      case MonitorElement::DQM_KIND_TH2S:
        {
          me.getTH2S()->Scale(factor);
          break;
        }
      case MonitorElement::DQM_KIND_TH2D:
        {
          me.getTH2D()->Scale(factor);
          break;
        }
      case MonitorElement::DQM_KIND_TH3F:
        {
          me.getTH3F()->Scale(factor);
          break;
        }
      case MonitorElement::DQM_KIND_TPROFILE:
        {
          me.getTProfile()->Scale(factor);
          break;
        }
      case MonitorElement::DQM_KIND_TPROFILE2D:
        {
          me.getTProfile2D()->Scale(factor);
          break;
        }
      default:
        if (verbose_ > 0)
          std::cout << " The DQM object '" << me.getFullname() << "' is not scalable object " << std::endl;
        continue;
      }
  }
}