ELstatistics.cc

Go to the documentation of this file.

//  ---------------------------------------------------------------------
//
// ELstatistics.cc
//
// History:
//   7/8/98     mf      Created
//   7/2/99     jv      Added noTerminationSummary() function
//   6/7/00     web     Reflect consolidation of ELdestination/X;
//                      consolidate ELstatistics/X
//   6/14/00    web     Remove GNU relic code
//   6/15/00    web     using -> USING
//   10/4/00    mf      filterModule() and excludeModule()
//   3/13/00    mf      statisticsMap()
//    4/4/01    mf      Simplify filter/exclude logic by useing base class
//                      method thisShouldBeIgnored().  Eliminate
//                      moduleOfinterest and moduleToexclude.
//  11/01/01    web     Remove last vestige of GNU relic code; reordered
//                      initializers to correspond to order of member
//                      declarations
//   1/17/06    mf  summary() for use in MessageLogger
//   8/16/07    mf  Changes to implement grouping of modules in specified
//          categories
//   6/19/08    mf  summaryForJobReport()
//
//  ---------------------------------------------------------------------

#include "FWCore/MessageService/src/ELstatistics.h"

#include "FWCore/MessageLogger/interface/ErrorObj.h"

#include <iostream>
#include <iomanip>
#include <sstream>
#include <ios>
#include <cassert>

// Possible Traces:
// #define ELstatisticsCONSTRUCTOR_TRACE
// #define ELstatsLOG_TRACE

namespace {
  std::string summarizeContext(const std::string& c) {
    if (c.substr(0, 4) != "Run:")
      return c;
    std::istringstream is(c);
    std::string runWord;
    int run;
    is >> runWord >> run;
    if (!is)
      return c;
    if (runWord != "Run:")
      return c;
    std::string eventWord;
    int event;
    is >> eventWord >> event;
    if (!is)
      return c;
    if (eventWord != "Event:")
      return c;
    std::ostringstream os;
    os << run << "/" << event;
    return os.str();
  }
}  // namespace

using namespace edm::messagelogger;

namespace edm {
  namespace service {

    // ----------------------------------------------------------------------
    // Constructors
    // ----------------------------------------------------------------------

    ELstatistics::ELstatistics()
        : ELdestination(),
          tableLimit(-1),
          stats(),
          updatedStats(false),
          termStream(std::cerr),
          printAtTermination(true) {
#ifdef ELstatisticsCONSTRUCTOR_TRACE
      std::cerr << "Constructor for ELstatistics()\n";
#endif

    }  // ELstatistics()

    ELstatistics::ELstatistics(std::ostream& osp)
        : ELdestination(), tableLimit(-1), stats(), updatedStats(false), termStream(osp), printAtTermination(true) {
#ifdef ELstatisticsCONSTRUCTOR_TRACE
      std::cerr << "Constructor for ELstatistics(osp)\n";
#endif

    }  // ELstatistics()

    ELstatistics::ELstatistics(int spaceLimit)
        : ELdestination(),
          tableLimit(spaceLimit),
          stats(),
          updatedStats(false),
          termStream(std::cerr),
          printAtTermination(true) {
#ifdef ELstatisticsCONSTRUCTOR_TRACE
      std::cerr << "Constructor for ELstatistics(spaceLimit)\n";
#endif

    }  // ELstatistics()

    ELstatistics::ELstatistics(int spaceLimit, std::ostream& osp)
        : ELdestination(),
          tableLimit(spaceLimit),
          stats(),
          updatedStats(false),
          termStream(osp),
          printAtTermination(true) {
#ifdef ELstatisticsCONSTRUCTOR_TRACE
      std::cerr << "Constructor for ELstatistics(spaceLimit,osp)\n";
#endif

    }  // ELstatistics()

    ELstatistics::ELstatistics(const ELstatistics& orig)
        : ELdestination(),
          tableLimit(orig.tableLimit),
          stats(orig.stats),
          updatedStats(orig.updatedStats),
          termStream(orig.termStream),
          printAtTermination(orig.printAtTermination) {
#ifdef ELstatisticsCONSTRUCTOR_TRACE
      std::cerr << "Copy constructor for ELstatistics()\n";
#endif

      ignoreMostModules = orig.ignoreMostModules;
      respondToThese = orig.respondToThese;
      respondToMostModules = orig.respondToMostModules;
      ignoreThese = orig.ignoreThese;

    }  // ELstatistics()

    ELstatistics::~ELstatistics() {
#ifdef ELstatisticsCONSTRUCTOR_TRACE
      std::cerr << "Destructor for ELstatistics\n";
#endif

      if (updatedStats && printAtTermination)
        summary(termStream, "Termination Summary");

    }  // ~ELstatistics()

    // ----------------------------------------------------------------------
    // Methods invoked by the ELadministrator
    // ----------------------------------------------------------------------

    bool ELstatistics::log(const edm::ErrorObj& msg) {
#ifdef ELstatsLOG_TRACE
      std::cerr << "  =:=:=: Log to an ELstatistics\n";
#endif

      // See if this message is to be counted.

      if (msg.xid().severity < threshold)
        return false;
      if (thisShouldBeIgnored(msg.xid().module))
        return false;

      // Account for this message, making a new table entry if needed:
      //
      ELmap_stats::iterator s = stats.find(msg.xid());
      if (s == stats.end()) {
        if (tableLimit < 0 || static_cast<int>(stats.size()) < tableLimit) {
          stats[msg.xid()] = StatsCount();
          s = stats.find(msg.xid());
        }
      }
#ifdef ELstatsLOG_TRACE
      std::cerr << "    =:=:=: Message accounted for in stats \n";
#endif
      if (s != stats.end()) {
        (*s).second.add(summarizeContext(msg.context()), msg.reactedTo());

        updatedStats = true;
#ifdef ELstatsLOG_TRACE
        std::cerr << "    =:=:=: Updated stats \n";
#endif
      }

      // For the purposes of telling whether any log destination has reacted
      // to the message, the statistics destination does not count:
      //

#ifdef ELstatsLOG_TRACE
      std::cerr << "  =:=:=: log(msg) done (stats) \n";
#endif

      return false;

    }  // log()

    void ELstatistics::clearSummary() {
      limits.zero();
      ELmap_stats::iterator s;
      for (s = stats.begin(); s != stats.end(); ++s) {
        (*s).second.n = 0;
        (*s).second.context1 = (*s).second.context2 = (*s).second.contextLast = "";
      }

    }  // clearSummary()

    void ELstatistics::wipe() {
      limits.wipe();
      stats.erase(stats.begin(), stats.end());  //stats.clear();

    }  // wipe()

    void ELstatistics::zero() { limits.zero(); }  // zero()

    std::string ELstatistics::formSummary(ELmap_stats& stats) {
      // Major changes 8/16/07 mf, including making this
      // a static member function instead of a free function

      using std::ios; /* _base ? */
      using std::left;
      using std::right;
      using std::setw;

      std::ostringstream s;
      int n = 0;

      // -----  Summary part I:
      //
      bool ftnote(false);

      struct part3 {
        long n, t;
        part3() : n(0L), t(0L) { ; }
      } p3[ELseverityLevel::nLevels];

      std::set<std::string>::iterator gcEnd = groupedCategories.end();
      std::set<std::string> gCats = groupedCategories;  // TEMP FOR DEBUGGING SANITY
      for (ELmap_stats::const_iterator i = stats.begin(); i != stats.end(); ++i) {
        // If this is a grouped category, wait till later to output its stats
        std::string cat = (*i).first.id;
        if (groupedCategories.find(cat) != gcEnd) {  // 8/16/07 mf
          continue;                                  // We will process these categories later
        }

        // -----  Emit new process and part I header, if needed:
        //
        if (n == 0) {
          s << "\n";
          s << " type     category        sev    module        "
               "subroutine        count    total\n"
            << " ---- -------------------- -- ---------------- "
               "----------------  -----    -----\n";
        }
        // -----  Emit detailed message information:
        //
        s << right << std::setw(5) << ++n << ' ' << left << std::setw(20) << (*i).first.id.substr(0, 20) << ' ' << left
          << std::setw(2) << (*i).first.severity.getSymbol() << ' ' << left << std::setw(16)
          << (*i).first.module.substr(0, 16) << ' ' << left << std::setw(16) << (*i).first.subroutine.substr(0, 16)
          << right << std::setw(7) << (*i).second.n << left << std::setw(1) << ((*i).second.ignoredFlag ? '*' : ' ')
          << right << std::setw(8) << (*i).second.aggregateN << '\n';
        ftnote = ftnote || (*i).second.ignoredFlag;

        // -----  Obtain information for Part III, below:
        //
        ELextendedID xid = (*i).first;
        p3[xid.severity.getLevel()].n += (*i).second.n;
        p3[xid.severity.getLevel()].t += (*i).second.aggregateN;
      }  // for i

      // ----- Part Ia:  The grouped categories
      for (std::set<std::string>::iterator g = groupedCategories.begin(); g != gcEnd; ++g) {
        int groupTotal = 0;
        int groupAggregateN = 0;
        ELseverityLevel severityLevel;
        bool groupIgnored = true;
        for (ELmap_stats::const_iterator i = stats.begin(); i != stats.end(); ++i) {
          if ((*i).first.id == *g) {
            if (groupTotal == 0)
              severityLevel = (*i).first.severity;
            groupIgnored &= (*i).second.ignoredFlag;
            groupAggregateN += (*i).second.aggregateN;
            ++groupTotal;
          }
        }  // for i
        if (groupTotal > 0) {
          // -----  Emit detailed message information:
          //
          s << right << std::setw(5) << ++n << ' ' << left << std::setw(20) << (*g).substr(0, 20) << ' ' << left
            << std::setw(2) << severityLevel.getSymbol() << ' ' << left << std::setw(16) << "  <Any Module>  " << ' '
            << left << std::setw(16) << "<Any Function>" << right << std::setw(7) << groupTotal << left << std::setw(1)
            << (groupIgnored ? '*' : ' ') << right << std::setw(8) << groupAggregateN << '\n';
          ftnote = ftnote || groupIgnored;

          // -----  Obtain information for Part III, below:
          //
          int lev = severityLevel.getLevel();
          p3[lev].n += groupTotal;
          p3[lev].t += groupAggregateN;
        }  // end if groupTotal>0
      }    // for g

      // -----  Provide footnote to part I, if needed:
      //
      if (ftnote)
        s << "\n* Some occurrences of this message"
             " were suppressed in all logs, due to limits.\n";

      // -----  Summary part II:
      //
      n = 0;
      for (ELmap_stats::const_iterator i = stats.begin(); i != stats.end(); ++i) {
        std::string cat = (*i).first.id;
        if (groupedCategories.find(cat) != gcEnd) {  // 8/16/07 mf
          continue;                                  // We will process these categories later
        }
        if (n == 0) {
          s << '\n'
            << " type    category    Examples: "
               "run/evt        run/evt          run/evt\n"
            << " ---- -------------------- ----"
               "------------ ---------------- ----------------\n";
        }
        s << right << std::setw(5) << ++n << ' ' << left << std::setw(20) << (*i).first.id.c_str() << ' ' << left
          << std::setw(16) << (*i).second.context1.c_str() << ' ' << left << std::setw(16)
          << (*i).second.context2.c_str() << ' ' << (*i).second.contextLast.c_str() << '\n';
      }  // for

      // -----  Summary part III:
      //
      s << "\nSeverity    # Occurrences   Total Occurrences\n"
        << "--------    -------------   -----------------\n";
      for (int k = 0; k < ELseverityLevel::nLevels; ++k) {
        if (p3[k].n != 0 || p3[k].t != 0) {
          s << left << std::setw(8) << ELseverityLevel(ELseverityLevel::ELsev_(k)).getName().c_str() << right
            << std::setw(17) << p3[k].n << right << std::setw(20) << p3[k].t << '\n';
        }
      }  // for

      return s.str();

    }  // formSummary()

    void ELstatistics::summary(std::ostream& os, std::string_view title) {
      os << title << std::endl << formSummary(stats) << std::flush;
      updatedStats = false;

    }  // summary()

    void ELstatistics::summary(unsigned long overfullWaitCount) {
      termStream << "\n=============================================\n\n"
                 << "MessageLogger Summary" << std::endl
                 << formSummary(stats) << std::endl
                 << "dropped waiting message count " << overfullWaitCount << std::endl
                 << std::flush;
      updatedStats = false;

    }  // summary()

    void ELstatistics::noTerminationSummary() { printAtTermination = false; }

    std::map<ELextendedID, StatsCount> ELstatistics::statisticsMap() const {
      return std::map<ELextendedID, StatsCount>(stats);
    }

    // 6/19/08 mf
    void ELstatistics::summaryForJobReport(std::map<std::string, double>& sm) {
      struct part3 {
        long n, t;
        part3() : n(0L), t(0L) { ; }
      } p3[ELseverityLevel::nLevels];

      std::set<std::string>::iterator gcEnd = groupedCategories.end();
      std::set<std::string> gCats = groupedCategories;  // TEMP FOR DEBUGGING SANITY

      // ----- Part I:  The ungrouped categories
      for (ELmap_stats::const_iterator i = stats.begin(); i != stats.end(); ++i) {
        // If this is a grouped category, wait till later to output its stats
        std::string cat = (*i).first.id;
        if (groupedCategories.find(cat) != gcEnd) {
          continue;  // We will process these categories later
        }

        // -----  Emit detailed message information:
        //
        std::ostringstream s;
        s << "Category_";
        std::string sevSymbol = (*i).first.severity.getSymbol();
        if (sevSymbol[0] == '-')
          sevSymbol = sevSymbol.substr(1);
        s << sevSymbol << "_" << (*i).first.id;
        int n = (*i).second.aggregateN;
        std::string catstr = s.str();
        if (sm.find(catstr) != sm.end()) {
          sm[catstr] += n;
        } else {
          sm[catstr] = n;
        }
        // -----  Obtain information for Part III, below:
        //
        ELextendedID xid = (*i).first;
        p3[xid.severity.getLevel()].n += (*i).second.n;
        p3[xid.severity.getLevel()].t += (*i).second.aggregateN;
      }  // for i

      // ----- Part Ia:  The grouped categories
      for (std::set<std::string>::iterator g = groupedCategories.begin(); g != gcEnd; ++g) {
        int groupTotal = 0;
        int groupAggregateN = 0;
        ELseverityLevel severityLevel;
        for (ELmap_stats::const_iterator i = stats.begin(); i != stats.end(); ++i) {
          if ((*i).first.id == *g) {
            if (groupTotal == 0)
              severityLevel = (*i).first.severity;
            groupAggregateN += (*i).second.aggregateN;
            ++groupTotal;
          }
        }  // for i
        if (groupTotal > 0) {
          // -----  Emit detailed message information:
          //
          std::ostringstream s;
          s << "Category_";
          std::string sevSymbol = severityLevel.getSymbol();
          if (sevSymbol[0] == '-')
            sevSymbol = sevSymbol.substr(1);
          s << sevSymbol << "_" << *g;
          int n = groupAggregateN;
          std::string catstr = s.str();
          if (sm.find(catstr) != sm.end()) {
            sm[catstr] += n;
          } else {
            sm[catstr] = n;
          }

          // -----  Obtain information for Part III, below:
          //
          int lev = severityLevel.getLevel();
          p3[lev].n += groupTotal;
          p3[lev].t += groupAggregateN;
        }  // end if groupTotal>0
      }    // for g

      // part II (sample event numbers) does not exist for the job report.

      // -----  Summary part III:
      //
      for (int k = 0; k < ELseverityLevel::nLevels; ++k) {
        //if ( p3[k].t != 0 )  {
        if (true) {
          std::string sevName;
          sevName = ELseverityLevel(ELseverityLevel::ELsev_(k)).getName();
          if (sevName == "Severe")
            sevName = "System";
          if (sevName == "Success")
            sevName = "Debug";
          sevName = std::string("Log") + sevName;
          sevName = dualLogName(sevName);
          if (sevName != "UnusedSeverity") {
            sm[sevName] = p3[k].t;
          }
        }
      }  // for k

    }  // summaryForJobReport()

    std::string ELstatistics::dualLogName(std::string const& s) {
      if (s == "LogDebug")
        return "LogDebug_LogTrace";
      if (s == "LogInfo")
        return "LogInfo_LogVerbatim";
      if (s == "LogWarning")
        return "LogWarnng_LogPrint";
      if (s == "LogError")
        return "LogError_LogProblem";
      if (s == "LogSystem")
        return "LogSystem_LogAbsolute";
      return "UnusedSeverity";
    }

    std::set<std::string> ELstatistics::groupedCategories;  // 8/16/07 mf

    void ELstatistics::noteGroupedCategory(std::string const& cat) { groupedCategories.insert(cat); }

  }  // end of namespace service
}  // end of namespace edm