HltDiff Class Reference

Public Member Functions
void	compare () const
	HltDiff ()
void	usage (std::ostream &out) const

Public Attributes
bool	csv_out
bool	debug
bool	file_check
bool	ignore_prescales
bool	json_out
unsigned int	max_events
std::vector< std::string >	new_files
std::string	new_process
std::vector< std::string >	old_files
std::string	old_process
std::string	output_file
bool	quiet
bool	root_out
unsigned int	verbose

Detailed Description

Definition at line 1239 of file hltDiff.cc.

Constructor & Destructor Documentation

HltDiff()

HltDiff::HltDiff ( )

inline

Definition at line 1256 of file hltDiff.cc.

      : old_files(0),
        old_process(""),
        new_files(0),
        new_process(""),
        max_events(1e9),
        ignore_prescales(true),
        csv_out(false),
        json_out(false),
        root_out(false),
        output_file(""),
        file_check(false),
        debug(false),
        quiet(false),
        verbose(0) {}

Member Function Documentation

compare()

void HltDiff::compare ( ) const

inline

Definition at line 1272 of file hltDiff.cc.

References edm::HLTGlobalStatus::accept(), EcnaPython_AdcPeg12_S1_10_R170298_1_0_150_Dee0::cerr, check_files(), universalConfigTemplate::collection, oniaPATMuonsWithTrigger_cff::collections, trigger::TriggerEvent::collectionTags(), counter, gather_cfg::cout, debug, edmPickEvents::event, event_state(), spr::find(), HLTCommonConfig::First, fwlite::Handle< T >::getByLabel(), getHLTConfigData(), mps_fire::i, edm::HLTGlobalStatus::index(), createfilelist::int, Invalid, fwlite::Handle< T >::isValid(), SiStripPI::max, writedatasetfile::max_events, SiStripPI::min, callgraph::module, HLTConfigInterface::moduleLabel(), HLTConfigInterface::moduleType(), submitPVValidationJobs::nEvents, or, electronStore::output_file, AlCaHLTBitMon_ParallelJobs::p, Pass, path_state(), Prescaled, prescaled_state(), print_detailed_path_state(), print_trigger_candidates(), print_trigger_collection(), HLTConfigInterface::processName(), fwlite::Handle< T >::product(), Ready, writedatasetfile::run, HLTCommonConfig::Second, HLTConfigInterface::size(), runEdmFileComparison::skipped, edm::HLTGlobalStatus::state(), AlCaHLTBitMon_QueryRunRegistry::string, strip_process_name(), edmLumisInFiles::summary, HLTConfigInterface::triggerIndex(), HLTConfigInterface::triggerName(), and verbose.

                       {
    std::shared_ptr<fwlite::ChainEvent> old_events;
    std::shared_ptr<fwlite::ChainEvent> new_events;

    if (not file_check or check_files(old_files))
      old_events = std::make_shared<fwlite::ChainEvent>(old_files);
    else
      return;

    if (new_files.size() == 1 and new_files[0] == "-")
      new_events = old_events;
    else if (not file_check or check_files(new_files))
      new_events = std::make_shared<fwlite::ChainEvent>(new_files);
    else
      return;

    // creating the structure holding data for JSON and ROOT output
    JsonOutputProducer json(json_out, output_file);

    json.configuration.prescales = ignore_prescales;
    // setting the old configuration
    json.configuration.o.process = old_process;
    json.configuration.o.files = old_files;
    json.configuration.o.extractFileBase();
    // setting the new configuration
    json.configuration.n.process = new_process;
    json.configuration.n.files = new_files;
    json.configuration.n.extractFileBase();

    // initialising configurations to be compared
    std::unique_ptr<HLTConfigDataEx> old_config_data;
    std::unique_ptr<HLTConfigDataEx> new_config_data;
    std::unique_ptr<HLTCommonConfig> common_config;
    HLTConfigInterface const* old_config = nullptr;
    HLTConfigInterface const* new_config = nullptr;

    unsigned int counter = 0;
    unsigned int skipped = 0;
    unsigned int affected = 0;
    bool new_run = true;
    std::vector<TriggerDiff> differences;

    // loop over the reference events
    const unsigned int nEvents = std::min((int)old_events->size(), (int)max_events);
    const unsigned int counter_denominator = std::max(1, int(nEvents / 10));
    for (old_events->toBegin(); not old_events->atEnd(); ++(*old_events)) {
      // printing progress on every 10%
      if (counter % (counter_denominator) == 0) {
        std::cout << "Processed events: " << counter << " out of " << nEvents << " ("
                  << 10 * counter / (counter_denominator) << "%)" << std::endl;
      }

      // seek the same event in the "new" files
      edm::EventID const& id = old_events->id();
      if (new_events != old_events and not new_events->to(id)) {
        if (debug)
          std::cerr << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event()
                    << ": not found in the 'new' files, skipping." << std::endl;
        ++skipped;
        continue;
      }

      // read the TriggerResults and TriggerEvent
      fwlite::Handle<edm::TriggerResults> old_results_h;
      edm::TriggerResults const* old_results = nullptr;
      old_results_h.getByLabel<fwlite::Event>(*old_events->event(), "TriggerResults", "", old_process.c_str());
      if (old_results_h.isValid())
        old_results = old_results_h.product();
      else {
        if (debug)
          std::cerr << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event()
                    << ": 'old' TriggerResults not found, skipping." << std::endl;
        continue;
      }

      fwlite::Handle<trigger::TriggerEvent> old_summary_h;
      trigger::TriggerEvent const* old_summary = nullptr;
      old_summary_h.getByLabel<fwlite::Event>(*old_events->event(), "hltTriggerSummaryAOD", "", old_process.c_str());
      if (old_summary_h.isValid())
        old_summary = old_summary_h.product();

      fwlite::Handle<edm::TriggerResults> new_results_h;
      edm::TriggerResults const* new_results = nullptr;
      new_results_h.getByLabel<fwlite::Event>(*new_events->event(), "TriggerResults", "", new_process.c_str());
      if (new_results_h.isValid())
        new_results = new_results_h.product();
      else {
        if (debug)
          std::cerr << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event()
                    << ": 'new' TriggerResults not found, skipping." << std::endl;
        continue;
      }

      fwlite::Handle<trigger::TriggerEvent> new_summary_h;
      trigger::TriggerEvent const* new_summary = nullptr;
      new_summary_h.getByLabel<fwlite::Event>(*new_events->event(), "hltTriggerSummaryAOD", "", new_process.c_str());
      if (new_summary_h.isValid())
        new_summary = new_summary_h.product();

      // initialise the trigger configuration
      if (new_run) {
        new_run = false;
        old_events->fillParameterSetRegistry();
        new_events->fillParameterSetRegistry();

        old_config_data = getHLTConfigData(*old_events->event(), old_process);
        new_config_data = getHLTConfigData(*new_events->event(), new_process);
        if (new_config_data->triggerNames() == old_config_data->triggerNames()) {
          old_config = old_config_data.get();
          new_config = new_config_data.get();
        } else {
          common_config = std::make_unique<HLTCommonConfig>(*old_config_data, *new_config_data);
          old_config = &common_config->getView(HLTCommonConfig::Index::First);
          new_config = &common_config->getView(HLTCommonConfig::Index::Second);
          std::cout << "Warning: old and new TriggerResults come from different HLT menus. Only the common "
                    << old_config->size() << " triggers are compared.\n"
                    << std::endl;
        }

        differences.clear();
        differences.resize(old_config->size());

        // adding the list of selected triggers to JSON output
        std::vector<std::string> states_str;
        for (int i = State::Ready; i != State::Invalid; i++)
          states_str.push_back(std::string(path_state(static_cast<State>(i))));
        json.vars.state = states_str;
        for (size_t triggerId = 0; triggerId < old_config->size(); ++triggerId) {
          json.vars.trigger.push_back(old_config->triggerName(triggerId));
          json.vars.trigger_passed_count.push_back(std::pair<int, int>(0, 0));
        }
        // getting names of triggers existing only in the old configuration
        for (auto const& it : old_config_data->triggerNames()) {
          if (std::find(json.vars.trigger.begin(), json.vars.trigger.end(), it) != json.vars.trigger.end())
            continue;
          json.configuration.o.skipped_triggers.push_back(it);
        }
        // getting names of triggers existing only in the new configuration
        for (auto const& it : new_config_data->triggerNames()) {
          if (std::find(json.vars.trigger.begin(), json.vars.trigger.end(), it) != json.vars.trigger.end())
            continue;
          json.configuration.n.skipped_triggers.push_back(it);
        }
      }

      // compare the TriggerResults
      bool needs_header = true;
      bool event_affected = false;
      for (unsigned int p = 0; p < old_config->size(); ++p) {
        // FIXME explicitly converting the indices is a hack, it should be properly encapsulated instead
        unsigned int old_index = old_config->triggerIndex(p);
        unsigned int new_index = new_config->triggerIndex(p);
        State old_state = prescaled_state(old_results->state(old_index), p, old_results->index(old_index), *old_config);
        State new_state = prescaled_state(new_results->state(new_index), p, new_results->index(new_index), *new_config);

        if (old_state == Pass) {
          ++differences.at(p).count;
        }
        if (old_state == Pass)
          ++json.vars.trigger_passed_count.at(p).first;
        if (new_state == Pass)
          ++json.vars.trigger_passed_count.at(p).second;

        bool trigger_affected = false;
        if (not ignore_prescales or (old_state != Prescaled and new_state != Prescaled)) {
          if (old_state == Pass and new_state != Pass) {
            ++differences.at(p).lost;
            trigger_affected = true;
          } else if (old_state != Pass and new_state == Pass) {
            ++differences.at(p).gained;
            trigger_affected = true;
          } else if (old_results->index(old_index) != new_results->index(new_index)) {
            ++differences.at(p).internal;
            trigger_affected = true;
          }
        }

        if (not trigger_affected)
          continue;

        event_affected = true;
        const unsigned int old_moduleIndex = old_results->index(old_index);
        const unsigned int new_moduleIndex = new_results->index(new_index);
        // storing the event to JSON, without any trigger results for the moment
        JsonOutputProducer::JsonEvent& event = json.pushEvent(id.run(), id.luminosityBlock(), id.event());
        JsonOutputProducer::JsonTriggerEventState& state = event.pushTrigger(p);
        state.o = json.eventState(old_state,
                                  old_moduleIndex,
                                  old_config->moduleLabel(p, old_moduleIndex),
                                  old_config->moduleType(p, old_moduleIndex));
        state.n = json.eventState(new_state,
                                  new_moduleIndex,
                                  new_config->moduleLabel(p, new_moduleIndex),
                                  new_config->moduleType(p, new_moduleIndex));

        if (verbose > 0) {
          if (needs_header) {
            needs_header = false;
            std::cout << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event() << ": "
                      << "old result is '" << event_state(old_results->accept()) << "', "
                      << "new result is '" << event_state(new_results->accept()) << "'" << std::endl;
          }
          // print the Trigger path and filter responsible for the discrepancy
          std::cout << "    Path " << old_config->triggerName(p) << ":\n"
                    << "        old state is ";
          print_detailed_path_state(std::cout, old_state, p, old_moduleIndex, *old_config);
          std::cout << ",\n"
                    << "        new state is ";
          print_detailed_path_state(std::cout, new_state, p, new_moduleIndex, *new_config);
          std::cout << std::endl;
        }
        if (verbose > 1 and old_summary and new_summary) {
          // print TriggerObjects for the filter responsible for the discrepancy
          unsigned int module = std::min(old_moduleIndex, new_moduleIndex);
          std::cout << "    Filter " << old_config->moduleLabel(p, module) << ":\n";
          std::cout << "        old trigger candidates:\n";
          print_trigger_candidates(std::cout,
                                   *old_summary,
                                   edm::InputTag(old_config->moduleLabel(p, module), "", old_config->processName()));
          std::cout << "        new trigger candidates:\n";
          print_trigger_candidates(std::cout,
                                   *new_summary,
                                   edm::InputTag(new_config->moduleLabel(p, module), "", new_config->processName()));
        }
        if (verbose > 0)
          std::cout << std::endl;
      }
      if (event_affected)
        ++affected;

      // compare the TriggerEvent
      if (event_affected and verbose > 2 and old_summary and new_summary) {
        std::map<std::string, std::pair<std::string, std::string>> collections;
        for (auto const& old_collection : old_summary->collectionTags())
          collections[strip_process_name(old_collection)].first = old_collection;
        for (auto const& new_collection : new_summary->collectionTags())
          collections[strip_process_name(new_collection)].second = new_collection;

        for (auto const& collection : collections) {
          std::cout << "    Collection " << collection.first << ":\n";
          std::cout << "        old trigger candidates:\n";
          print_trigger_collection(std::cout, *old_summary, collection.second.first);
          std::cout << "        new trigger candidates:\n";
          print_trigger_collection(std::cout, *new_summary, collection.second.second);
          std::cout << std::endl;
        }
      }

      ++counter;
      if (nEvents and counter >= nEvents)
        break;
    }

    json.configuration.events = counter;

    if (not counter) {
      std::cout << "There are no common events between the old and new files";
      if (skipped)
        std::cout << ", " << skipped << " events were skipped";
      std::cout << "." << std::endl;
    } else {
      std::cout << "Found " << counter << " matching events, out of which " << affected
                << " have different HLT results";
      if (skipped)
        std::cout << ", " << skipped << " events were skipped";
      std::cout << "\n" << std::endl;
    }
    // Printing the summary of affected triggers with affected-event counts
    if (!quiet and old_config) {
      bool summaryHeaderPrinted = false;
      for (size_t p = 0; p < old_config->size(); ++p) {
        if (differences.at(p).total() < 1)
          continue;
        if (!summaryHeaderPrinted)
          std::cout << std::setw(12) << "Events" << std::setw(12) << "Accepted" << std::setw(12) << "Gained"
                    << std::setw(12) << "Lost" << std::setw(12) << "Other"
                    << "  "
                    << "Trigger" << std::endl;
        std::cout << std::setw(12) << counter << differences.at(p) << "  " << old_config->triggerName(p) << std::endl;
        summaryHeaderPrinted = true;
      }
    }

    // writing all the required output
    json.write();  // to JSON file for interactive visualisation
    SummaryOutputProducer summary(json, this->root_out, this->csv_out);
    summary.write();  // to ROOT file for fast validation with static plots
  }

usage()

void HltDiff::usage ( std::ostream &  out ) const

inline

Definition at line 1561 of file hltDiff.cc.

References MillePedeFileConverter_cfg::out.

                                    {
    out << "\
usage: hltDiff -o|--old-files FILE1.ROOT [FILE2.ROOT ...] [-O|--old-process LABEL[:INSTANCE[:PROCESS]]]\n\
               -n|--new-files FILE1.ROOT [FILE2.ROOT ...] [-N|--new-process LABEL[:INSTANCE[:PROCESS]]]\n\
               [-m|--max-events MAXEVENTS] [-p|--prescales] [-c|--csv-output] [-j|--json-output]\n\
               [-r|--root-output] [-f|--file-check] [-d|--debug] [-q|--quiet] [-v|--verbose]\n\
               [-h|--help] [-F|--output-file] FILE_NAME\n\
\n\
  -o|--old-files FILE1.ROOT [FILE2.ROOT ...]\n\
      input file(s) with the old (reference) trigger results\n\
\n\
  -O|--old-process PROCESS\n\
      process name of the collection with the old (reference) trigger results\n\
      default: take the 'TriggerResults' from the last process\n\
\n\
  -n|--new-files FILE1.ROOT [FILE2.ROOT ...]\n\
      input file(s) with the new trigger results to be compared with the reference\n\
      to read these from a different collection in the same files as\n\
      the reference, use '-n -' and specify the collection with -N (see below)\n\
\n\
  -N|--new-process PROCESS\n\
      process name of the collection with the new (reference) trigger results\n\
      default: take the 'TriggerResults' from the last process\n\
\n\
  -m|--max-events MAXEVENTS\n\
      compare only the first MAXEVENTS events\n\
      default: compare all the events in the original (reference) files\n\
\n\
  -p|--prescales\n\
      do not ignore differences caused by HLTPrescaler modules\n\
\n\
  -c|--csv-output\n\
      produce comparison results in a CSV format\n\
\n\
  -j|--json-output\n\
      produce comparison results in a JSON format\n\
\n\
  -r|--root-output\n\
      produce comparison results as histograms in a ROOT file\n\
\n\
  -F|--output-file FILE_NAME\n\
      combine all RUNs to files with the specified custom name: FILE_NAME.json, FILE_NAME.root\n\
      default: a separate output file will be produced for each RUN with names suitable for the DQM GUI\n\
\n\
  -f|--file-check\n\
      check existence of every old and new file before running the comparison\n\
      safer if files are run for the first time, but can cause a substantial delay\n\
\n\
  -d|--debug\n\
      display messages about missing events and collections\n\
\n\
  -q|--quiet\n\
      don't display summary printout with the list of affected trigger paths\n\
\n\
  -v|--verbose LEVEL\n\
      set verbosity level:\n\
      1: event-by-event comparison results\n\
      2: + print the trigger candidates of the affected filters\n\
      3: + print all the trigger candidates for the affected events\n\
      default: 1\n\
\n\
  -h|--help\n\
      print this help message, and exit"
        << std::endl;
  }

Member Data Documentation

csv_out

bool HltDiff::csv_out

Definition at line 1247 of file hltDiff.cc.

debug

bool HltDiff::debug

Definition at line 1252 of file hltDiff.cc.

Referenced by rrapi.RRApi::dprint(), rrapi.RRApi::get(), runTauIdMVA.TauIDEmbedder::loadMVA_WPs_run2_2017(), and runTauIdMVA.TauIDEmbedder::runTauID().

file_check

bool HltDiff::file_check

Definition at line 1251 of file hltDiff.cc.

ignore_prescales

bool HltDiff::ignore_prescales

Definition at line 1246 of file hltDiff.cc.

json_out

bool HltDiff::json_out

Definition at line 1248 of file hltDiff.cc.

max_events

unsigned int HltDiff::max_events

Definition at line 1245 of file hltDiff.cc.

new_files

std::vector<std::string> HltDiff::new_files

Definition at line 1243 of file hltDiff.cc.

new_process

std::string HltDiff::new_process

Definition at line 1244 of file hltDiff.cc.

old_files

std::vector<std::string> HltDiff::old_files

Definition at line 1241 of file hltDiff.cc.

old_process

std::string HltDiff::old_process

Definition at line 1242 of file hltDiff.cc.

output_file

std::string HltDiff::output_file

Definition at line 1250 of file hltDiff.cc.

Referenced by DTWorkflow.DTWorkflow::prepare_common_write().

quiet

bool HltDiff::quiet

Definition at line 1253 of file hltDiff.cc.

root_out

bool HltDiff::root_out

Definition at line 1249 of file hltDiff.cc.

verbose

unsigned int HltDiff::verbose

Definition at line 1254 of file hltDiff.cc.

Referenced by python.diff_provenance.difference::list_diff(), python.diffProv.difference::list_diff(), python.diff_provenance.difference::module_diff(), python.diffProv.difference::module_diff(), python.diff_provenance.difference::onefilemodules(), python.diffProv.difference::onefilemodules(), core.TriggerMatchAnalyzer.TriggerMatchAnalyzer::process(), core.SkimAnalyzerCount.SkimAnalyzerCount::process(), and objects.VertexAnalyzer.VertexAnalyzer::process().