HltDiff Class Reference

Public Member Functions
void	compare () const
	HltDiff ()
void	usage (std::ostream &out) const

Public Attributes
bool	debug
bool	file_check
bool	ignore_prescales
bool	json_out
unsigned int	max_events
std::vector< std::string >	new_files
std::string	new_process
std::vector< std::string >	old_files
std::string	old_process
std::string	output_file
bool	root_out
unsigned int	verbose

Detailed Description

Definition at line 1177 of file hltDiff.cc.

Constructor & Destructor Documentation

HltDiff::HltDiff ( )

inline

Definition at line 1194 of file hltDiff.cc.

            :
    old_files(0),
    old_process(""),
    new_files(0),
    new_process(""),
    max_events(1e9),
    ignore_prescales(true),
    json_out(false),
    root_out(false),
    output_file(""),
    file_check(false),
    debug(false),
    verbose(0) {}

Member Function Documentation

void HltDiff::compare ( ) const

inline

Definition at line 1208 of file hltDiff.cc.

References edm::HLTGlobalStatus::accept(), ecal_dqm_sourceclient-live_cfg::cerr, check_files(), runEdmFileComparison::collection, trigger::TriggerEvent::collectionTags(), JsonOutputProducer::configuration, counter, gather_cfg::cout, debug, event(), event_state(), JsonOutputProducer::JsonConfiguration::events, JsonOutputProducer::eventState(), JsonOutputProducer::JsonConfigurationBlock::extractFileBase(), JsonOutputProducer::JsonConfigurationBlock::files, spr::find(), HLTCommonConfig::First, fwlite::Handle< T >::getByLabel(), getHLTConfigData(), HLTCommonConfig::HLTCommonConfig(), i, edm::HLTGlobalStatus::index(), Invalid, fwlite::Handle< T >::isValid(), L1Trigger_dataformats::json, min(), HLTConfigInterface::moduleLabel(), HLTConfigInterface::moduleType(), JsonOutputProducer::JsonConfiguration::n, JsonOutputProducer::JsonTriggerEventState::n, nEvents, JsonOutputProducer::JsonConfiguration::o, JsonOutputProducer::JsonTriggerEventState::o, or, electronStore::output_file, AlCaHLTBitMon_ParallelJobs::p, Pass, path_state(), Prescaled, prescaled_state(), JsonOutputProducer::JsonConfiguration::prescales, print_detailed_path_state(), print_trigger_candidates(), print_trigger_collection(), JsonOutputProducer::JsonConfigurationBlock::process, HLTConfigInterface::processName(), fwlite::Handle< T >::product(), JsonOutputProducer::pushEvent(), Ready, DTTTrigCorrFirst::run, HLTCommonConfig::Second, HLTConfigInterface::size(), JsonOutputProducer::JsonConfigurationBlock::skipped_triggers, edm::HLTGlobalStatus::state(), JsonOutputProducer::JsonVars::state, AlCaHLTBitMon_QueryRunRegistry::string, strip_process_name(), edmLumisInFiles::summary, JsonOutputProducer::JsonVars::trigger, JsonOutputProducer::JsonVars::trigger_passed_count, HLTConfigInterface::triggerIndex(), HLTConfigInterface::triggerName(), JsonOutputProducer::vars, JsonOutputProducer::write(), and SummaryOutputProducer::write().

Referenced by main().

                       {
    std::shared_ptr<fwlite::ChainEvent> old_events;
    std::shared_ptr<fwlite::ChainEvent> new_events;

    if (not file_check or check_files(old_files))
      old_events = std::make_shared<fwlite::ChainEvent>(old_files);
    else
      return;

    if (new_files.size() == 1 and new_files[0] == "-")
      new_events = old_events;
    else if (not file_check or check_files(new_files))
      new_events = std::make_shared<fwlite::ChainEvent>(new_files);
    else
      return;

    // creating the structure holding data for JSON and ROOT output
    JsonOutputProducer json(json_out, output_file);

    json.configuration.prescales = ignore_prescales;
    // setting the old configuration
    json.configuration.o.process = old_process;
    json.configuration.o.files = old_files;
    json.configuration.o.extractFileBase();
    // setting the new configuration
    json.configuration.n.process = new_process;
    json.configuration.n.files = new_files;
    json.configuration.n.extractFileBase();

    // initialising configurations to be compared
    std::unique_ptr<HLTConfigDataEx> old_config_data;
    std::unique_ptr<HLTConfigDataEx> new_config_data;
    std::unique_ptr<HLTCommonConfig> common_config;
    HLTConfigInterface const * old_config = nullptr;
    HLTConfigInterface const * new_config = nullptr;

    unsigned int counter = 0;
    unsigned int skipped = 0;
    unsigned int affected = 0;
    bool new_run = true;
    std::vector<TriggerDiff> differences;

    // loop over the reference events
    const unsigned int nEvents = std::min((int)old_events->size(), (int)max_events);
    for (old_events->toBegin(); not old_events->atEnd(); ++(*old_events)) {
      // printing progress on every 10%
      if (counter%(nEvents/10) == 0) {
        printf("Processed events: %d out of %d (%d%%)\n", (int)counter, (int)nEvents, 10*counter/(nEvents/10));
      }

      // seek the same event in the "new" files
      edm::EventID const& id = old_events->id();
      if (new_events != old_events and not new_events->to(id)) {
        if (debug)
          std::cerr << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event() << ": not found in the 'new' files, skipping." << std::endl;
        ++skipped;
        continue;
      }

      // read the TriggerResults and TriggerEvent
      fwlite::Handle<edm::TriggerResults> old_results_h;
      edm::TriggerResults const * old_results = nullptr;
      old_results_h.getByLabel<fwlite::Event>(* old_events->event(), "TriggerResults", "", old_process.c_str());
      if (old_results_h.isValid())
        old_results = old_results_h.product();
      else {
        if (debug)
          std::cerr << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event() << ": 'old' TriggerResults not found, skipping." << std::endl;
        continue;
      }

      fwlite::Handle<trigger::TriggerEvent> old_summary_h;
      trigger::TriggerEvent const * old_summary = nullptr;
      old_summary_h.getByLabel<fwlite::Event>(* old_events->event(), "hltTriggerSummaryAOD", "", old_process.c_str());
      if (old_summary_h.isValid())
        old_summary = old_summary_h.product();

      fwlite::Handle<edm::TriggerResults> new_results_h;
      edm::TriggerResults const * new_results = nullptr;
      new_results_h.getByLabel<fwlite::Event>(* new_events->event(), "TriggerResults", "", new_process.c_str());
      if (new_results_h.isValid())
        new_results = new_results_h.product();
      else {
        if (debug)
          std::cerr << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event() << ": 'new' TriggerResults not found, skipping." << std::endl;
        continue;
      }

      fwlite::Handle<trigger::TriggerEvent> new_summary_h;
      trigger::TriggerEvent const * new_summary = nullptr;
      new_summary_h.getByLabel<fwlite::Event>(* new_events->event(), "hltTriggerSummaryAOD", "", new_process.c_str());
      if (new_summary_h.isValid())
        new_summary = new_summary_h.product();

      // initialise the trigger configuration
      if (new_run) {
        new_run = false;
        old_events->fillParameterSetRegistry();
        new_events->fillParameterSetRegistry();

        old_config_data = getHLTConfigData(* old_events->event(), old_process);
        new_config_data = getHLTConfigData(* new_events->event(), new_process);
        if (new_config_data->triggerNames() == old_config_data->triggerNames()) {
          old_config = old_config_data.get();
          new_config = new_config_data.get();
        } else {
          common_config = std::unique_ptr<HLTCommonConfig>(new HLTCommonConfig(*old_config_data, *new_config_data));
          old_config = & common_config->getView(HLTCommonConfig::Index::First);
          new_config = & common_config->getView(HLTCommonConfig::Index::Second);
          std::cout << "Warning: old and new TriggerResults come from different HLT menus. Only the common " << old_config->size() << " triggers are compared.\n" << std::endl;
        }

        differences.clear();
        differences.resize(old_config->size());

        // adding the list of selected triggers to JSON output
        std::vector<std::string> states_str;
        for (int i = State::Ready; i != State::Invalid; i++)
          states_str.push_back(std::string(path_state(static_cast<State>(i))));
        json.vars.state = states_str;
        for (size_t triggerId = 0; triggerId < old_config->size(); ++triggerId) {
          json.vars.trigger.push_back(old_config->triggerName(triggerId));
          json.vars.trigger_passed_count.push_back(std::pair<int, int>(0,0));
        }
        // getting names of triggers existing only in the old configuration
        for (std::vector<std::string>::const_iterator it = old_config_data->triggerNames().begin(); it != old_config_data->triggerNames().end(); ++it) {
          if (std::find(json.vars.trigger.begin(), json.vars.trigger.end(), *it) != json.vars.trigger.end()) continue;
          json.configuration.o.skipped_triggers.push_back(*it);
        }
        // getting names of triggers existing only in the new configuration
        for (std::vector<std::string>::const_iterator it = new_config_data->triggerNames().begin(); it != new_config_data->triggerNames().end(); ++it) {
          if (std::find(json.vars.trigger.begin(), json.vars.trigger.end(), *it) != json.vars.trigger.end()) continue;
          json.configuration.n.skipped_triggers.push_back(*it);
        }
      }

      // compare the TriggerResults
      bool needs_header = true;
      bool event_affected = false;
      for (unsigned int p = 0; p < old_config->size(); ++p) {
        // FIXME explicitly converting the indices is a hack, it should be properly encapsulated instead
        unsigned int old_index = old_config->triggerIndex(p);
        unsigned int new_index = new_config->triggerIndex(p);
        State old_state = prescaled_state(old_results->state(old_index), p, old_results->index(old_index), * old_config);
        State new_state = prescaled_state(new_results->state(new_index), p, new_results->index(new_index), * new_config);

        if (old_state == Pass) {
          ++differences[p].count;
        }
        if (old_state == Pass)
          ++json.vars.trigger_passed_count.at(p).first;
        if (new_state == Pass)
          ++json.vars.trigger_passed_count.at(p).second;

        bool trigger_affected = false;
        if (not ignore_prescales or (old_state != Prescaled and new_state != Prescaled)) {
          if (old_state == Pass and new_state != Pass) {
            ++differences[p].lost;
            trigger_affected = true;
          } else if (old_state != Pass and new_state == Pass) {
            ++differences[p].gained;
            trigger_affected = true;
          } else if (old_results->index(old_index) != new_results->index(new_index)) {
            ++differences[p].internal;
            trigger_affected = true;
          }
        }

        if (not trigger_affected) continue;
        
        event_affected = true;
        const unsigned int old_moduleIndex = old_results->index(old_index);
        const unsigned int new_moduleIndex = new_results->index(new_index);
        // storing the event to JSON, without any trigger results for the moment
        JsonOutputProducer::JsonEvent& event = json.pushEvent(id.run(), id.luminosityBlock(), id.event());
        JsonOutputProducer::JsonTriggerEventState& state = event.pushTrigger(p);
        state.o = json.eventState(old_state, old_moduleIndex, old_config->moduleLabel(p, old_moduleIndex), old_config->moduleType(p, old_moduleIndex));
        state.n = json.eventState(new_state, new_moduleIndex, new_config->moduleLabel(p, new_moduleIndex), new_config->moduleType(p, new_moduleIndex));

        if (verbose > 0) {
          if (needs_header) {
            needs_header = false;
            std::cout << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event() << ": "
                      << "old result is '" << event_state(old_results->accept()) << "', "
                      << "new result is '" << event_state(new_results->accept()) << "'"
                      << std::endl;
          }
          // print the Trigger path and filter responsible for the discrepancy
          std::cout << "    Path " << old_config->triggerName(p) << ":\n"
                    << "        old state is ";
          print_detailed_path_state(std::cout, old_state, p, old_moduleIndex, * old_config);
          std::cout << ",\n"
                    << "        new state is ";
          print_detailed_path_state(std::cout, new_state, p, new_moduleIndex, * new_config);
          std::cout << std::endl;
        }
        if (verbose > 1 and old_summary and new_summary) {
          // print TriggerObjects for the filter responsible for the discrepancy
          unsigned int module = std::min(old_moduleIndex, new_moduleIndex);
          std::cout << "    Filter " << old_config->moduleLabel(p, module) << ":\n";
          std::cout << "        old trigger candidates:\n";
          print_trigger_candidates(std::cout, * old_summary, edm::InputTag(old_config->moduleLabel(p, module), "", old_config->processName()));
          std::cout << "        new trigger candidates:\n";
          print_trigger_candidates(std::cout, * new_summary, edm::InputTag(new_config->moduleLabel(p, module), "", new_config->processName()));
        }
        if (verbose > 0)
          std::cout << std::endl;
      }
      if (event_affected)
        ++affected;

      // compare the TriggerEvent
      if (event_affected and verbose > 2 and old_summary and new_summary) {
        std::map<std::string, std::pair<std::string, std::string>> collections;
        for (auto const & old_collection: old_summary->collectionTags())
          collections[strip_process_name(old_collection)].first  = old_collection;
        for (auto const & new_collection: new_summary->collectionTags())
          collections[strip_process_name(new_collection)].second = new_collection;

        for (auto const & collection: collections) {
          std::cout << "    Collection " << collection.first << ":\n";
          std::cout << "        old trigger candidates:\n";
          print_trigger_collection(std::cout, * old_summary, collection.second.first);
          std::cout << "        new trigger candidates:\n";
          print_trigger_collection(std::cout, * new_summary, collection.second.second);
          std::cout << std::endl;
        }
      }

      ++counter;
      if (nEvents and counter >= nEvents)
        break;
    }
    
    json.configuration.events = counter;

    if (not counter) {
      std::cout << "There are no common events between the old and new files";
      if (skipped)
        std::cout << ", " << skipped << " events were skipped";
      std::cout <<  "." << std::endl;
    } else {
      std::cout << "Found " << counter << " matching events, out of which " << affected << " have different HLT results";
      if (skipped)
        std::cout << ", " << skipped << " events were skipped";
      std::cout << "\nSee more in the files listed below...\n" << std::endl;
    }

    // writing all the required output
    json.write();   // to JSON file for interactive visualisation
    SummaryOutputProducer summary(json, this->root_out, true);
    summary.write();   // to ROOT file for fast validation with static plots
  }

void HltDiff::usage ( std::ostream &  out ) const

inline

Definition at line 1462 of file hltDiff.cc.

Referenced by main().

                                     {
    out << "\
usage: hltDiff -o|--old-files FILE1.ROOT [FILE2.ROOT ...] [-O|--old-process LABEL[:INSTANCE[:PROCESS]]]\n\
               -n|--new-files FILE1.ROOT [FILE2.ROOT ...] [-N|--new-process LABEL[:INSTANCE[:PROCESS]]]\n\
               [-m|--max-events MAXEVENTS] [-p|--prescales] [-j|--json-output] OUTPUT_FILE.JSON\n\
               [-r|--root-output] OUTPUT_FILE.ROOT [-f|--file-check] [-d|--debug] [-v|--verbose] [-h|--help]\n\
\n\
  -o|--old-files FILE1.ROOT [FILE2.ROOT ...]\n\
      input file(s) with the old (reference) trigger results\n\
\n\
  -O|--old-process PROCESS\n\
      process name of the collection with the old (reference) trigger results\n\
      default: take the 'TriggerResults' from the last process\n\
\n\
  -n|--new-files FILE1.ROOT [FILE2.ROOT ...]\n\
      input file(s) with the new trigger results to be compared with the reference\n\
      to read these from a different collection in the same files as\n\
      the reference, use '-n -' and specify the collection with -N (see below)\n\
\n\
  -N|--new-process PROCESS\n\
      process name of the collection with the new (reference) trigger results\n\
      default: take the 'TriggerResults' from the last process\n\
\n\
  -m|--max-events MAXEVENTS\n\
      compare only the first MAXEVENTS events\n\
      default: compare all the events in the original (reference) files\n\
\n\
  -p|--prescales\n\
      do not ignore differences caused by HLTPrescaler modules\n\
\n\
  -j|--json-output\n\
      produce comparison results in a JSON format\n\
\n\
  -r|--root-output\n\
      produce comparison results as histograms in a ROOT file\n\
\n\
  -F|--output-file FILE_NAME\n\
      combine all RUNs to files with the specified custom name: FILE_NAME.json, FILE_NAME.root\n\
      default: a separate output file will be produced for each RUN with names suitable for the DQM GUI\n\
\n\
  -f|--file-check\n\
      check existence of every old and new file before running the comparison\n\
      safer if files are run for the first time, but can cause a substantial delay\n\
\n\
  -d|--debug\n\
      display messages about missing events and collectiions\n\
\n\
  -v|--verbose LEVEL\n\
      set verbosity level:\n\
      1: event-by-event comparison results\n\
      2: + print the trigger candidates of the affected filters\n\
      3: + print all the trigger candidates for the affected events\n\
      default: 1\n\
\n\
  -h|--help\n\
      print this help message, and exit" << std::endl;
  }

Member Data Documentation

bool HltDiff::debug

Definition at line 1191 of file hltDiff.cc.

Referenced by rrapi.RRApi::dprint(), rrapi.RRApi::get(), and main().

bool HltDiff::file_check

Definition at line 1190 of file hltDiff.cc.

Referenced by main().

bool HltDiff::ignore_prescales

Definition at line 1186 of file hltDiff.cc.

Referenced by main().

bool HltDiff::json_out

Definition at line 1187 of file hltDiff.cc.

Referenced by main().

unsigned int HltDiff::max_events

Definition at line 1185 of file hltDiff.cc.

Referenced by main().

std::vector<std::string> HltDiff::new_files

Definition at line 1183 of file hltDiff.cc.

Referenced by main().

std::string HltDiff::new_process

Definition at line 1184 of file hltDiff.cc.

Referenced by main().

std::vector<std::string> HltDiff::old_files

Definition at line 1181 of file hltDiff.cc.

Referenced by main().

std::string HltDiff::old_process

Definition at line 1182 of file hltDiff.cc.

Referenced by main().

std::string HltDiff::output_file

Definition at line 1189 of file hltDiff.cc.

Referenced by main().

bool HltDiff::root_out

Definition at line 1188 of file hltDiff.cc.

Referenced by main().

unsigned int HltDiff::verbose

Definition at line 1192 of file hltDiff.cc.

Referenced by python.diff_provenance.difference::list_diff(), python.diffProv.difference::list_diff(), main(), python.diff_provenance.difference::module_diff(), python.diffProv.difference::module_diff(), python.diff_provenance.difference::onefilemodules(), python.diffProv.difference::onefilemodules(), core.TriggerMatchAnalyzer.TriggerMatchAnalyzer::process(), core.SkimAnalyzerCount.SkimAnalyzerCount::process(), and objects.VertexAnalyzer.VertexAnalyzer::process().