CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
List of all members | Public Member Functions | Public Attributes
HltDiff Class Reference

Public Member Functions

void compare () const
 
 HltDiff ()
 
void usage (std::ostream &out) const
 

Public Attributes

bool csv_out
 
bool debug
 
bool file_check
 
bool ignore_prescales
 
bool json_out
 
unsigned int max_events
 
std::vector< std::string > new_files
 
std::string new_process
 
std::vector< std::string > old_files
 
std::string old_process
 
std::string output_file
 
bool quiet
 
bool root_out
 
unsigned int verbose
 

Detailed Description

Definition at line 1239 of file hltDiff.cc.

Constructor & Destructor Documentation

◆ HltDiff()

HltDiff::HltDiff ( )
inline

Definition at line 1256 of file hltDiff.cc.

1257  : old_files(0),
1258  old_process(""),
1259  new_files(0),
1260  new_process(""),
1261  max_events(1e9),
1262  ignore_prescales(true),
1263  csv_out(false),
1264  json_out(false),
1265  root_out(false),
1266  output_file(""),
1267  file_check(false),
1268  debug(false),
1269  quiet(false),
1270  verbose(0) {}
bool debug
Definition: hltDiff.cc:1252
bool root_out
Definition: hltDiff.cc:1249
bool file_check
Definition: hltDiff.cc:1251
unsigned int verbose
Definition: hltDiff.cc:1254
bool quiet
Definition: hltDiff.cc:1253
std::vector< std::string > old_files
Definition: hltDiff.cc:1241
std::string output_file
Definition: hltDiff.cc:1250
std::vector< std::string > new_files
Definition: hltDiff.cc:1243
bool csv_out
Definition: hltDiff.cc:1247
bool ignore_prescales
Definition: hltDiff.cc:1246
bool json_out
Definition: hltDiff.cc:1248
unsigned int max_events
Definition: hltDiff.cc:1245
std::string new_process
Definition: hltDiff.cc:1244
std::string old_process
Definition: hltDiff.cc:1242

Member Function Documentation

◆ compare()

void HltDiff::compare ( ) const
inline

Definition at line 1272 of file hltDiff.cc.

References edm::HLTGlobalStatus::accept(), DMR_cfg::cerr, check_files(), universalConfigTemplate::collection, oniaPATMuonsWithTrigger_cff::collections, trigger::TriggerEvent::collectionTags(), counter, gather_cfg::cout, debug, edmPickEvents::event, event_state(), spr::find(), HLTCommonConfig::First, fwlite::Handle< T >::getByLabel(), getHLTConfigData(), mps_fire::i, edm::HLTGlobalStatus::index(), createfilelist::int, Invalid, fwlite::Handle< T >::isValid(), SiStripPI::max, writedatasetfile::max_events, SiStripPI::min, callgraph::module, HLTConfigInterface::moduleLabel(), HLTConfigInterface::moduleType(), createIOVlist::nEvents, or, electronStore::output_file, AlCaHLTBitMon_ParallelJobs::p, Pass, path_state(), Prescaled, prescaled_state(), print_detailed_path_state(), print_trigger_candidates(), print_trigger_collection(), HLTConfigInterface::processName(), fwlite::Handle< T >::product(), Ready, writedatasetfile::run, HLTCommonConfig::Second, HLTConfigInterface::size(), runEdmFileComparison::skipped, edm::HLTGlobalStatus::state(), AlCaHLTBitMon_QueryRunRegistry::string, strip_process_name(), edmLumisInFiles::summary, HLTConfigInterface::triggerIndex(), HLTConfigInterface::triggerName(), and verbose.

1272  {
1273  std::shared_ptr<fwlite::ChainEvent> old_events;
1274  std::shared_ptr<fwlite::ChainEvent> new_events;
1275 
1277  old_events = std::make_shared<fwlite::ChainEvent>(old_files);
1278  else
1279  return;
1280 
1281  if (new_files.size() == 1 and new_files[0] == "-")
1282  new_events = old_events;
1283  else if (not file_check or check_files(new_files))
1284  new_events = std::make_shared<fwlite::ChainEvent>(new_files);
1285  else
1286  return;
1287 
1288  // creating the structure holding data for JSON and ROOT output
1290 
1291  json.configuration.prescales = ignore_prescales;
1292  // setting the old configuration
1293  json.configuration.o.process = old_process;
1294  json.configuration.o.files = old_files;
1295  json.configuration.o.extractFileBase();
1296  // setting the new configuration
1297  json.configuration.n.process = new_process;
1298  json.configuration.n.files = new_files;
1299  json.configuration.n.extractFileBase();
1300 
1301  // initialising configurations to be compared
1302  std::unique_ptr<HLTConfigDataEx> old_config_data;
1303  std::unique_ptr<HLTConfigDataEx> new_config_data;
1304  std::unique_ptr<HLTCommonConfig> common_config;
1305  HLTConfigInterface const* old_config = nullptr;
1306  HLTConfigInterface const* new_config = nullptr;
1307 
1308  unsigned int counter = 0;
1309  unsigned int skipped = 0;
1310  unsigned int affected = 0;
1311  bool new_run = true;
1312  std::vector<TriggerDiff> differences;
1313 
1314  // loop over the reference events
1315  const unsigned int nEvents = std::min((int)old_events->size(), (int)max_events);
1316  const unsigned int counter_denominator = std::max(1, int(nEvents / 10));
1317  for (old_events->toBegin(); not old_events->atEnd(); ++(*old_events)) {
1318  // printing progress on every 10%
1319  if (counter % (counter_denominator) == 0) {
1320  std::cout << "Processed events: " << counter << " out of " << nEvents << " ("
1321  << 10 * counter / (counter_denominator) << "%)" << std::endl;
1322  }
1323 
1324  // seek the same event in the "new" files
1325  edm::EventID const& id = old_events->id();
1326  if (new_events != old_events and not new_events->to(id)) {
1327  if (debug)
1328  std::cerr << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event()
1329  << ": not found in the 'new' files, skipping." << std::endl;
1330  ++skipped;
1331  continue;
1332  }
1333 
1334  // read the TriggerResults and TriggerEvent
1336  edm::TriggerResults const* old_results = nullptr;
1337  old_results_h.getByLabel<fwlite::Event>(*old_events->event(), "TriggerResults", "", old_process.c_str());
1338  if (old_results_h.isValid())
1339  old_results = old_results_h.product();
1340  else {
1341  if (debug)
1342  std::cerr << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event()
1343  << ": 'old' TriggerResults not found, skipping." << std::endl;
1344  continue;
1345  }
1346 
1348  trigger::TriggerEvent const* old_summary = nullptr;
1349  old_summary_h.getByLabel<fwlite::Event>(*old_events->event(), "hltTriggerSummaryAOD", "", old_process.c_str());
1350  if (old_summary_h.isValid())
1351  old_summary = old_summary_h.product();
1352 
1354  edm::TriggerResults const* new_results = nullptr;
1355  new_results_h.getByLabel<fwlite::Event>(*new_events->event(), "TriggerResults", "", new_process.c_str());
1356  if (new_results_h.isValid())
1357  new_results = new_results_h.product();
1358  else {
1359  if (debug)
1360  std::cerr << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event()
1361  << ": 'new' TriggerResults not found, skipping." << std::endl;
1362  continue;
1363  }
1364 
1366  trigger::TriggerEvent const* new_summary = nullptr;
1367  new_summary_h.getByLabel<fwlite::Event>(*new_events->event(), "hltTriggerSummaryAOD", "", new_process.c_str());
1368  if (new_summary_h.isValid())
1369  new_summary = new_summary_h.product();
1370 
1371  // initialise the trigger configuration
1372  if (new_run) {
1373  new_run = false;
1374  old_events->fillParameterSetRegistry();
1375  new_events->fillParameterSetRegistry();
1376 
1377  old_config_data = getHLTConfigData(*old_events->event(), old_process);
1378  new_config_data = getHLTConfigData(*new_events->event(), new_process);
1379  if (new_config_data->triggerNames() == old_config_data->triggerNames()) {
1380  old_config = old_config_data.get();
1381  new_config = new_config_data.get();
1382  } else {
1383  common_config = std::make_unique<HLTCommonConfig>(*old_config_data, *new_config_data);
1384  old_config = &common_config->getView(HLTCommonConfig::Index::First);
1385  new_config = &common_config->getView(HLTCommonConfig::Index::Second);
1386  std::cout << "Warning: old and new TriggerResults come from different HLT menus. Only the common "
1387  << old_config->size() << " triggers are compared.\n"
1388  << std::endl;
1389  }
1390 
1391  differences.clear();
1392  differences.resize(old_config->size());
1393 
1394  // adding the list of selected triggers to JSON output
1395  std::vector<std::string> states_str;
1396  for (int i = State::Ready; i != State::Invalid; i++)
1397  states_str.push_back(std::string(path_state(static_cast<State>(i))));
1398  json.vars.state = states_str;
1399  for (size_t triggerId = 0; triggerId < old_config->size(); ++triggerId) {
1400  json.vars.trigger.push_back(old_config->triggerName(triggerId));
1401  json.vars.trigger_passed_count.push_back(std::pair<int, int>(0, 0));
1402  }
1403  // getting names of triggers existing only in the old configuration
1404  for (auto const& it : old_config_data->triggerNames()) {
1405  if (std::find(json.vars.trigger.begin(), json.vars.trigger.end(), it) != json.vars.trigger.end())
1406  continue;
1407  json.configuration.o.skipped_triggers.push_back(it);
1408  }
1409  // getting names of triggers existing only in the new configuration
1410  for (auto const& it : new_config_data->triggerNames()) {
1411  if (std::find(json.vars.trigger.begin(), json.vars.trigger.end(), it) != json.vars.trigger.end())
1412  continue;
1413  json.configuration.n.skipped_triggers.push_back(it);
1414  }
1415  }
1416 
1417  // compare the TriggerResults
1418  bool needs_header = true;
1419  bool event_affected = false;
1420  for (unsigned int p = 0; p < old_config->size(); ++p) {
1421  // FIXME explicitly converting the indices is a hack, it should be properly encapsulated instead
1422  unsigned int old_index = old_config->triggerIndex(p);
1423  unsigned int new_index = new_config->triggerIndex(p);
1424  State old_state = prescaled_state(old_results->state(old_index), p, old_results->index(old_index), *old_config);
1425  State new_state = prescaled_state(new_results->state(new_index), p, new_results->index(new_index), *new_config);
1426 
1427  if (old_state == Pass) {
1428  ++differences.at(p).count;
1429  }
1430  if (old_state == Pass)
1431  ++json.vars.trigger_passed_count.at(p).first;
1432  if (new_state == Pass)
1433  ++json.vars.trigger_passed_count.at(p).second;
1434 
1435  bool trigger_affected = false;
1436  if (not ignore_prescales or (old_state != Prescaled and new_state != Prescaled)) {
1437  if (old_state == Pass and new_state != Pass) {
1438  ++differences.at(p).lost;
1439  trigger_affected = true;
1440  } else if (old_state != Pass and new_state == Pass) {
1441  ++differences.at(p).gained;
1442  trigger_affected = true;
1443  } else if (old_results->index(old_index) != new_results->index(new_index)) {
1444  ++differences.at(p).internal;
1445  trigger_affected = true;
1446  }
1447  }
1448 
1449  if (not trigger_affected)
1450  continue;
1451 
1452  event_affected = true;
1453  const unsigned int old_moduleIndex = old_results->index(old_index);
1454  const unsigned int new_moduleIndex = new_results->index(new_index);
1455  // storing the event to JSON, without any trigger results for the moment
1456  JsonOutputProducer::JsonEvent& event = json.pushEvent(id.run(), id.luminosityBlock(), id.event());
1457  JsonOutputProducer::JsonTriggerEventState& state = event.pushTrigger(p);
1458  state.o = json.eventState(old_state,
1459  old_moduleIndex,
1460  old_config->moduleLabel(p, old_moduleIndex),
1461  old_config->moduleType(p, old_moduleIndex));
1462  state.n = json.eventState(new_state,
1463  new_moduleIndex,
1464  new_config->moduleLabel(p, new_moduleIndex),
1465  new_config->moduleType(p, new_moduleIndex));
1466 
1467  if (verbose > 0) {
1468  if (needs_header) {
1469  needs_header = false;
1470  std::cout << "run " << id.run() << ", lumi " << id.luminosityBlock() << ", event " << id.event() << ": "
1471  << "old result is '" << event_state(old_results->accept()) << "', "
1472  << "new result is '" << event_state(new_results->accept()) << "'" << std::endl;
1473  }
1474  // print the Trigger path and filter responsible for the discrepancy
1475  std::cout << " Path " << old_config->triggerName(p) << ":\n"
1476  << " old state is ";
1477  print_detailed_path_state(std::cout, old_state, p, old_moduleIndex, *old_config);
1478  std::cout << ",\n"
1479  << " new state is ";
1480  print_detailed_path_state(std::cout, new_state, p, new_moduleIndex, *new_config);
1481  std::cout << std::endl;
1482  }
1483  if (verbose > 1 and old_summary and new_summary) {
1484  // print TriggerObjects for the filter responsible for the discrepancy
1485  unsigned int module = std::min(old_moduleIndex, new_moduleIndex);
1486  std::cout << " Filter " << old_config->moduleLabel(p, module) << ":\n";
1487  std::cout << " old trigger candidates:\n";
1489  *old_summary,
1490  edm::InputTag(old_config->moduleLabel(p, module), "", old_config->processName()));
1491  std::cout << " new trigger candidates:\n";
1493  *new_summary,
1494  edm::InputTag(new_config->moduleLabel(p, module), "", new_config->processName()));
1495  }
1496  if (verbose > 0)
1497  std::cout << std::endl;
1498  }
1499  if (event_affected)
1500  ++affected;
1501 
1502  // compare the TriggerEvent
1503  if (event_affected and verbose > 2 and old_summary and new_summary) {
1504  std::map<std::string, std::pair<std::string, std::string>> collections;
1505  for (auto const& old_collection : old_summary->collectionTags())
1506  collections[strip_process_name(old_collection)].first = old_collection;
1507  for (auto const& new_collection : new_summary->collectionTags())
1508  collections[strip_process_name(new_collection)].second = new_collection;
1509 
1510  for (auto const& collection : collections) {
1511  std::cout << " Collection " << collection.first << ":\n";
1512  std::cout << " old trigger candidates:\n";
1513  print_trigger_collection(std::cout, *old_summary, collection.second.first);
1514  std::cout << " new trigger candidates:\n";
1515  print_trigger_collection(std::cout, *new_summary, collection.second.second);
1516  std::cout << std::endl;
1517  }
1518  }
1519 
1520  ++counter;
1521  if (nEvents and counter >= nEvents)
1522  break;
1523  }
1524 
1525  json.configuration.events = counter;
1526 
1527  if (not counter) {
1528  std::cout << "There are no common events between the old and new files";
1529  if (skipped)
1530  std::cout << ", " << skipped << " events were skipped";
1531  std::cout << "." << std::endl;
1532  } else {
1533  std::cout << "Found " << counter << " matching events, out of which " << affected
1534  << " have different HLT results";
1535  if (skipped)
1536  std::cout << ", " << skipped << " events were skipped";
1537  std::cout << "\n" << std::endl;
1538  }
1539  // Printing the summary of affected triggers with affected-event counts
1540  if (!quiet and old_config) {
1541  bool summaryHeaderPrinted = false;
1542  for (size_t p = 0; p < old_config->size(); ++p) {
1543  if (differences.at(p).total() < 1)
1544  continue;
1545  if (!summaryHeaderPrinted)
1546  std::cout << std::setw(12) << "Events" << std::setw(12) << "Accepted" << std::setw(12) << "Gained"
1547  << std::setw(12) << "Lost" << std::setw(12) << "Other"
1548  << " "
1549  << "Trigger" << std::endl;
1550  std::cout << std::setw(12) << counter << differences.at(p) << " " << old_config->triggerName(p) << std::endl;
1551  summaryHeaderPrinted = true;
1552  }
1553  }
1554 
1555  // writing all the required output
1556  json.write(); // to JSON file for interactive visualisation
1558  summary.write(); // to ROOT file for fast validation with static plots
1559  }
unsigned int index(const unsigned int i) const
Get index (slot position) of module giving the decision of the ith path.
virtual std::string const & moduleLabel(unsigned int trigger, unsigned int module) const =0
bool accept() const
Has at least one path accepted the event?
hlt::HLTState state(const unsigned int i) const
Get status of ith path.
const std::vector< std::string > & collectionTags() const
Definition: TriggerEvent.h:99
bool check_files(std::vector< std::string > const &files)
Definition: hltDiff.cc:1229
The single EDProduct to be saved for each event (AOD case)
Definition: TriggerEvent.h:25
bool debug
Definition: hltDiff.cc:1252
bool root_out
Definition: hltDiff.cc:1249
Definition: hltDiff.cc:243
void print_trigger_candidates(std::ostream &out, trigger::TriggerEvent const &summary, edm::InputTag const &filter)
Definition: hltDiff.cc:297
nlohmann::json json
bool file_check
Definition: hltDiff.cc:1251
T const * product() const
Definition: Handle.h:65
unsigned int verbose
Definition: hltDiff.cc:1254
void print_detailed_path_state(std::ostream &out, State state, int path, int module, HLTConfigInterface const &config)
Definition: hltDiff.cc:286
const char * event_state(bool state)
Definition: hltDiff.cc:113
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:19
bool isValid() const
Definition: Handle.h:60
bool quiet
Definition: hltDiff.cc:1253
std::vector< std::string > old_files
Definition: hltDiff.cc:1241
State prescaled_state(int state, int path, int module, HLTConfigInterface const &config)
Definition: hltDiff.cc:259
std::string output_file
Definition: hltDiff.cc:1250
std::unique_ptr< HLTConfigDataEx > getHLTConfigData(fwlite::EventBase const &event, std::string process)
Definition: hltDiff.cc:362
The Signals That Services Can Subscribe To This is based on ActivityRegistry and is current per Services can connect to the signals distributed by the ActivityRegistry in order to monitor the activity of the application Each possible callback has some defined which we here list in angle e< void, edm::EventID const &, edm::Timestamp const & > We also list in braces which AR_WATCH_USING_METHOD_ is used for those or
Definition: Activities.doc:12
std::vector< std::string > new_files
Definition: hltDiff.cc:1243
bool csv_out
Definition: hltDiff.cc:1247
virtual std::string const & moduleType(unsigned int trigger, unsigned int module) const =0
bool ignore_prescales
Definition: hltDiff.cc:1246
bool json_out
Definition: hltDiff.cc:1248
std::string strip_process_name(std::string const &s)
Definition: hltDiff.cc:272
virtual unsigned int triggerIndex(unsigned int trigger) const =0
void print_trigger_collection(std::ostream &out, trigger::TriggerEvent const &summary, std::string const &tag)
Definition: hltDiff.cc:327
virtual std::string const & processName() const =0
void getByLabel(const P &iP, const char *iModuleLabel, const char *iProductInstanceLabel=nullptr, const char *iProcessLabel=nullptr)
Definition: Handle.h:100
const char * path_state(State state)
Definition: hltDiff.cc:250
unsigned int max_events
Definition: hltDiff.cc:1245
static std::atomic< unsigned int > counter
std::string new_process
Definition: hltDiff.cc:1244
std::string old_process
Definition: hltDiff.cc:1242

◆ usage()

void HltDiff::usage ( std::ostream &  out) const
inline

Definition at line 1561 of file hltDiff.cc.

References MillePedeFileConverter_cfg::out.

1561  {
1562  out << "\
1563 usage: hltDiff -o|--old-files FILE1.ROOT [FILE2.ROOT ...] [-O|--old-process LABEL[:INSTANCE[:PROCESS]]]\n\
1564  -n|--new-files FILE1.ROOT [FILE2.ROOT ...] [-N|--new-process LABEL[:INSTANCE[:PROCESS]]]\n\
1565  [-m|--max-events MAXEVENTS] [-p|--prescales] [-c|--csv-output] [-j|--json-output]\n\
1566  [-r|--root-output] [-f|--file-check] [-d|--debug] [-q|--quiet] [-v|--verbose]\n\
1567  [-h|--help] [-F|--output-file] FILE_NAME\n\
1568 \n\
1569  -o|--old-files FILE1.ROOT [FILE2.ROOT ...]\n\
1570  input file(s) with the old (reference) trigger results\n\
1571 \n\
1572  -O|--old-process PROCESS\n\
1573  process name of the collection with the old (reference) trigger results\n\
1574  default: take the 'TriggerResults' from the last process\n\
1575 \n\
1576  -n|--new-files FILE1.ROOT [FILE2.ROOT ...]\n\
1577  input file(s) with the new trigger results to be compared with the reference\n\
1578  to read these from a different collection in the same files as\n\
1579  the reference, use '-n -' and specify the collection with -N (see below)\n\
1580 \n\
1581  -N|--new-process PROCESS\n\
1582  process name of the collection with the new (reference) trigger results\n\
1583  default: take the 'TriggerResults' from the last process\n\
1584 \n\
1585  -m|--max-events MAXEVENTS\n\
1586  compare only the first MAXEVENTS events\n\
1587  default: compare all the events in the original (reference) files\n\
1588 \n\
1589  -p|--prescales\n\
1590  do not ignore differences caused by HLTPrescaler modules\n\
1591 \n\
1592  -c|--csv-output\n\
1593  produce comparison results in a CSV format\n\
1594 \n\
1595  -j|--json-output\n\
1596  produce comparison results in a JSON format\n\
1597 \n\
1598  -r|--root-output\n\
1599  produce comparison results as histograms in a ROOT file\n\
1600 \n\
1601  -F|--output-file FILE_NAME\n\
1602  combine all RUNs to files with the specified custom name: FILE_NAME.json, FILE_NAME.root\n\
1603  default: a separate output file will be produced for each RUN with names suitable for the DQM GUI\n\
1604 \n\
1605  -f|--file-check\n\
1606  check existence of every old and new file before running the comparison\n\
1607  safer if files are run for the first time, but can cause a substantial delay\n\
1608 \n\
1609  -d|--debug\n\
1610  display messages about missing events and collections\n\
1611 \n\
1612  -q|--quiet\n\
1613  don't display summary printout with the list of affected trigger paths\n\
1614 \n\
1615  -v|--verbose LEVEL\n\
1616  set verbosity level:\n\
1617  1: event-by-event comparison results\n\
1618  2: + print the trigger candidates of the affected filters\n\
1619  3: + print all the trigger candidates for the affected events\n\
1620  default: 1\n\
1621 \n\
1622  -h|--help\n\
1623  print this help message, and exit"
1624  << std::endl;
1625  }

Member Data Documentation

◆ csv_out

bool HltDiff::csv_out

Definition at line 1247 of file hltDiff.cc.

◆ debug

bool HltDiff::debug

◆ file_check

bool HltDiff::file_check

Definition at line 1251 of file hltDiff.cc.

◆ ignore_prescales

bool HltDiff::ignore_prescales

Definition at line 1246 of file hltDiff.cc.

◆ json_out

bool HltDiff::json_out

Definition at line 1248 of file hltDiff.cc.

◆ max_events

unsigned int HltDiff::max_events

Definition at line 1245 of file hltDiff.cc.

◆ new_files

std::vector<std::string> HltDiff::new_files

Definition at line 1243 of file hltDiff.cc.

◆ new_process

std::string HltDiff::new_process

Definition at line 1244 of file hltDiff.cc.

◆ old_files

std::vector<std::string> HltDiff::old_files

Definition at line 1241 of file hltDiff.cc.

◆ old_process

std::string HltDiff::old_process

Definition at line 1242 of file hltDiff.cc.

◆ output_file

std::string HltDiff::output_file

Definition at line 1250 of file hltDiff.cc.

Referenced by DTWorkflow.DTWorkflow::prepare_common_write().

◆ quiet

bool HltDiff::quiet

Definition at line 1253 of file hltDiff.cc.

◆ root_out

bool HltDiff::root_out

Definition at line 1249 of file hltDiff.cc.

◆ verbose

unsigned int HltDiff::verbose