FEDErrors.cc

Go to the documentation of this file.

#include "FWCore/Utilities/interface/Exception.h"
#include "FWCore/MessageLogger/interface/MessageLogger.h"

#include "DataFormats/FEDRawData/interface/FEDNumbering.h"
#include "DataFormats/DetId/interface/DetId.h"
#include "DataFormats/TrackerCommon/interface/TrackerTopology.h"

#include "EventFilter/SiStripRawToDigi/interface/PipeAddrToTimeLookupTable.h"

#include "DQM/SiStripMonitorHardware/interface/HistogramBase.hh"
#include "DQM/SiStripMonitorHardware/interface/FEDErrors.hh"
#include "DQM/SiStripCommon/interface/TkHistoMap.h"

FEDErrors::FEDErrors() {
  //initialiseLumiBlock();
  initialiseEvent();
}

FEDErrors::~FEDErrors() {}

void FEDErrors::initialiseLumiBlock() {}

void FEDErrors::initialiseEvent() {
  fedID_ = 0;
  failUnpackerFEDCheck_ = false;

  connected_.clear();
  detid_.clear();
  nChInModule_.clear();

  subDetId_.clear();

  lFedCounter_.nFEDErrors = 0;
  lFedCounter_.nDAQProblems = 0;
  lFedCounter_.nFEDsWithFEProblems = 0;
  lFedCounter_.nCorruptBuffers = 0;
  lFedCounter_.nBadChannels = 0;
  lFedCounter_.nBadActiveChannels = 0;
  lFedCounter_.nFEDsWithFEOverflows = 0;
  lFedCounter_.nFEDsWithFEBadMajorityAddresses = 0;
  lFedCounter_.nFEDsWithMissingFEs = 0;
  lFedCounter_.nTotalBadChannels = 0;
  lFedCounter_.nTotalBadActiveChannels = 0;

  lChCounter_.nNotConnected = 0;
  lChCounter_.nUnlocked = 0;
  lChCounter_.nOutOfSync = 0;
  lChCounter_.nAPVStatusBit = 0;
  lChCounter_.nAPVError = 0;
  lChCounter_.nAPVAddressError = 0;

  feCounter_.nFEOverflows = 0;
  feCounter_.nFEBadMajorityAddresses = 0;
  feCounter_.nFEMissing = 0;

  fedErrors_.HasCabledChannels = false;
  fedErrors_.DataPresent = false;
  fedErrors_.DataMissing = false;
  fedErrors_.InvalidBuffers = false;
  fedErrors_.BadFEDCRCs = false;
  fedErrors_.BadDAQCRCs = false;
  fedErrors_.BadIDs = false;
  fedErrors_.BadDAQPacket = false;
  fedErrors_.CorruptBuffer = false;
  fedErrors_.FEsOverflow = false;
  fedErrors_.FEsMissing = false;
  fedErrors_.FEsBadMajorityAddress = false;
  fedErrors_.BadChannelStatusBit = false;
  fedErrors_.BadActiveChannelStatusBit = false;

  feErrors_.clear();

  chErrorsDetailed_.clear();

  apvErrors_.clear();

  chErrors_.clear();

  eventProp_.deltaBX = 0;
}

void FEDErrors::initialiseFED(const unsigned int aFedID,
                              const SiStripFedCabling* aCabling,
                              const TrackerTopology* tTopo,
                              const bool initVars) {
  fedID_ = aFedID;
  failUnpackerFEDCheck_ = false;

  //initialise first.  if no channel connected in one FE, subdetid =
  //0.  in the loop on channels, if at least one channel is connected
  //and has a valid ID, the subdet value will be changed to the right
  //one.
  if (initVars) {
    subDetId_.resize(sistrip::FEUNITS_PER_FED, 0);

    connected_.resize(sistrip::FEDCH_PER_FED, false);
    detid_.resize(sistrip::FEDCH_PER_FED, 0);
    nChInModule_.resize(sistrip::FEDCH_PER_FED, 0);

    for (unsigned int iCh = 0; iCh < sistrip::FEDCH_PER_FED; iCh++) {
      const FedChannelConnection& lConnection = aCabling->fedConnection(fedID_, iCh);
      connected_[iCh] = lConnection.isConnected();
      detid_[iCh] = lConnection.detId();
      nChInModule_[iCh] = lConnection.nApvPairs();

      unsigned short lFeNumber = static_cast<unsigned int>(iCh / sistrip::FEDCH_PER_FEUNIT);
      unsigned int lDetid = detid_[iCh];

      if (lDetid && lDetid != sistrip::invalid32_ && connected_[iCh]) {
        unsigned int lSubid = 6;
        // 3=TIB, 4=TID, 5=TOB, 6=TEC (TECB here)
        switch (DetId(lDetid).subdetId()) {
          case 3:
            lSubid = 2;  //TIB
            break;

          case 4: {
            if (tTopo->tidSide(lDetid) == 2)
              lSubid = 4;  //TIDF
            else
              lSubid = 3;  //TIDB
            break;
          }

          case 5:
            lSubid = 5;  //TOB
            break;

          case 6: {
            if (tTopo->tecSide(lDetid) == 2)
              lSubid = 1;  //TECF
            else
              lSubid = 0;  //TECB
            break;
          }

          default:
            lSubid = 6;
            break;
        }
        subDetId_[lFeNumber] = lSubid;
        //if (iCh%12==0) std::cout << fedID_ << " " << lFeNumber << " " << subDetId_[lFeNumber] << std::endl;
      }
    }

    feCounter_.nFEOverflows = 0;
    feCounter_.nFEBadMajorityAddresses = 0;
    feCounter_.nFEMissing = 0;

    fedErrors_.HasCabledChannels = false;
    fedErrors_.DataPresent = false;
    fedErrors_.DataMissing = false;
    fedErrors_.InvalidBuffers = false;
    fedErrors_.BadFEDCRCs = false;
    fedErrors_.BadDAQCRCs = false;
    fedErrors_.BadIDs = false;
    fedErrors_.BadDAQPacket = false;
    fedErrors_.CorruptBuffer = false;
    fedErrors_.FEsOverflow = false;
    fedErrors_.FEsMissing = false;
    fedErrors_.FEsBadMajorityAddress = false;
    fedErrors_.BadChannelStatusBit = false;
    fedErrors_.BadActiveChannelStatusBit = false;

    feErrors_.clear();

    chErrorsDetailed_.clear();

    apvErrors_.clear();

    chErrors_.clear();
  }
}

bool FEDErrors::checkDataPresent(const FEDRawData& aFedData) {
  if (!aFedData.size() || !aFedData.data()) {
    for (unsigned int iCh = 0; iCh < sistrip::FEDCH_PER_FED; iCh++) {
      if (connected_[iCh]) {
        fedErrors_.HasCabledChannels = true;
        fedErrors_.DataMissing = true;
        return false;
      }
    }
    fedErrors_.DataMissing = true;
    fedErrors_.HasCabledChannels = false;
    return false;
  } else {
    fedErrors_.DataPresent = true;
    for (unsigned int iCh = 0; iCh < sistrip::FEDCH_PER_FED; iCh++) {
      if (connected_[iCh]) {
        fedErrors_.HasCabledChannels = true;
        break;
      }
    }
    return true;
  }
}

bool FEDErrors::failUnpackerFEDCheck() { return failUnpackerFEDCheck_; }

bool FEDErrors::fillFatalFEDErrors(const FEDRawData& aFedData, const unsigned int aPrintDebug) {
  const auto st_buffer = sistrip::preconstructCheckFEDBufferBase(aFedData);
  if (sistrip::FEDBufferStatusCode::SUCCESS != st_buffer) {
    fedErrors_.InvalidBuffers = true;
    failUnpackerFEDCheck_ = true;
    //don't check anything else if the buffer is invalid
    return false;
  }
  const sistrip::FEDBufferBase buffer{aFedData};

  //CRC checks
  //if CRC fails then don't continue as if the buffer has been corrupted in DAQ then anything else could be invalid
  if (!buffer.checkNoSlinkCRCError()) {
    fedErrors_.BadFEDCRCs = true;
    return false;
  } else if (!buffer.checkCRC()) {
    failUnpackerFEDCheck_ = true;
    fedErrors_.BadDAQCRCs = true;
    return false;
  }
  //next check that it is a SiStrip buffer
  //if not then stop checks
  if (!buffer.checkSourceIDs() || !buffer.checkNoUnexpectedSourceID()) {
    fedErrors_.BadIDs = true;
    return false;
  }
  //if so then do DAQ header/trailer checks
  //if these fail then buffer may be incomplete and checking contents doesn't make sense
  else if (!buffer.doDAQHeaderAndTrailerChecks()) {
    failUnpackerFEDCheck_ = true;
    fedErrors_.BadDAQPacket = true;
    return false;
  }

  //now do checks on header
  //check that tracker special header is consistent
  if (!(buffer.checkBufferFormat() && buffer.checkHeaderType() && buffer.checkReadoutMode())) {
    failUnpackerFEDCheck_ = true;
    fedErrors_.InvalidBuffers = true;
    //do not return false if debug printout of the buffer done below...
    if (!printDebug() || aPrintDebug < 3)
      return false;
  }

  //FE unit overflows
  if (!buffer.checkNoFEOverflows()) {
    failUnpackerFEDCheck_ = true;
    fedErrors_.FEsOverflow = true;
    //do not return false if debug printout of the buffer done below...
    if (!printDebug() || aPrintDebug < 3)
      return false;
  }

  return true;
}

bool FEDErrors::fillCorruptBuffer(const sistrip::FEDBuffer& aBuffer) {
  //corrupt buffer checks
  if (!(aBuffer.checkChannelLengthsMatchBufferLength() && aBuffer.checkChannelPacketCodes() &&
        aBuffer.checkFEUnitLengths())) {
    fedErrors_.CorruptBuffer = true;

    return false;
  }

  return true;
}

float FEDErrors::fillNonFatalFEDErrors(const sistrip::FEDBuffer* aBuffer, const SiStripFedCabling* aCabling) {
  unsigned int lBadChans = 0;
  unsigned int lTotChans = 0;
  for (unsigned int iCh = 0; iCh < sistrip::FEDCH_PER_FED; iCh++) {  //loop on channels
    bool lIsConnected = false;
    if (aCabling) {
      const FedChannelConnection& lConnection = aCabling->fedConnection(fedID_, iCh);
      lIsConnected = lConnection.isConnected();
    } else
      lIsConnected = connected_[iCh];

    if (!lIsConnected)
      continue;
    lTotChans++;
    if (!aBuffer->channelGood(iCh, true))
      lBadChans++;
  }

  return static_cast<float>(lBadChans * 1.0 / lTotChans);
}

bool FEDErrors::fillFEDErrors(const FEDRawData& aFedData,
                              bool& aFullDebug,
                              const unsigned int aPrintDebug,
                              unsigned int& aCounterMonitoring,
                              unsigned int& aCounterUnpacker,
                              const bool aDoMeds,
                              MonitorElement* aMedianHist0,
                              MonitorElement* aMedianHist1,
                              const bool aDoFEMaj,
                              std::vector<std::vector<std::pair<unsigned int, unsigned int> > >& aFeMajFrac) {
  //try to construct the basic buffer object (do not check payload)
  //if this fails then count it as an invalid buffer and stop checks since we can't understand things like buffer ordering

  if (!fillFatalFEDErrors(aFedData, aPrintDebug))
    return false;

  //need to construct full object to go any further
  const auto st_buffer = sistrip::preconstructCheckFEDBuffer(aFedData, true);
  if (sistrip::FEDBufferStatusCode::SUCCESS != st_buffer) {
    throw cms::Exception("FEDBuffer") << st_buffer << " (check debug output for more details)";
  }
  sistrip::FEDBuffer buffer{aFedData, true};
  buffer.findChannels();  // no need to check the status, also bad buffers are allowed

  //fill remaining unpackerFEDcheck
  if (!buffer.checkChannelLengths())
    failUnpackerFEDCheck_ = true;

  //payload checks, only if none of the above error occured
  if (!this->anyFEDErrors()) {
    bool lCorruptCheck = fillCorruptBuffer(buffer);
    if (aPrintDebug > 1 && !lCorruptCheck) {
      edm::LogWarning("SiStripMonitorHardware")
          << "CorruptBuffer check failed for FED " << fedID_ << std::endl
          << " -- buffer->checkChannelLengthsMatchBufferLength() = " << buffer.checkChannelLengthsMatchBufferLength()
          << std::endl
          << " -- buffer->checkChannelPacketCodes() = " << buffer.checkChannelPacketCodes() << std::endl
          << " -- buffer->checkFEUnitLengths() = " << buffer.checkFEUnitLengths() << std::endl;
    }

    //corruptBuffer concerns the payload: header info should still be reliable...
    //so analyze FE and channels to fill histograms.

    //fe check...
    fillFEErrors(buffer, aDoFEMaj, aFeMajFrac);

    //channel checks
    fillChannelErrors(
        buffer, aFullDebug, aPrintDebug, aCounterMonitoring, aCounterUnpacker, aDoMeds, aMedianHist0, aMedianHist1);
  }

  if (printDebug() && aPrintDebug > 2) {
    const sistrip::FEDBufferBase& debugBuffer = buffer;

    std::vector<FEDErrors::APVLevelErrors>& lChVec = getAPVLevelErrors();
    std::ostringstream debugStream;
    if (!lChVec.empty()) {
      std::sort(lChVec.begin(), lChVec.end());
      debugStream << "[FEDErrors] Cabled channels which had errors: ";

      for (unsigned int iBadCh(0); iBadCh < lChVec.size(); iBadCh++) {
        print(lChVec[iBadCh], debugStream);
      }
      debugStream << std::endl;
      debugStream << "[FEDErrors] Active (have been locked in at least one event) cabled channels which had errors: ";
      for (unsigned int iBadCh(0); iBadCh < lChVec.size(); iBadCh++) {
        if ((lChVec[iBadCh]).IsActive)
          print(lChVec[iBadCh], debugStream);
      }
    }
    debugStream << debugBuffer << std::endl;
    debugBuffer.dump(debugStream);
    debugStream << std::endl;
    edm::LogInfo("SiStripMonitorHardware") << "[FEDErrors] Errors found in FED " << fedID_;
    edm::LogVerbatim("SiStripMonitorHardware") << debugStream.str();
  }

  return !(anyFEDErrors());
}

bool FEDErrors::fillFEErrors(const sistrip::FEDBuffer& aBuffer,
                             const bool aDoFEMaj,
                             std::vector<std::vector<std::pair<unsigned int, unsigned int> > >& aFeMajFrac) {
  bool foundOverflow = false;
  bool foundBadMajority = false;
  bool foundMissing = false;
  for (unsigned int iFE = 0; iFE < sistrip::FEUNITS_PER_FED; iFE++) {
    FEDErrors::FELevelErrors lFeErr;
    lFeErr.FeID = iFE;
    lFeErr.SubDetID = subDetId_[iFE];
    lFeErr.Overflow = false;
    lFeErr.Missing = false;
    lFeErr.BadMajorityAddress = false;
    lFeErr.TimeDifference = 0;
    lFeErr.Apve = 0;
    lFeErr.FeMaj = 0;
    //check for cabled channels
    bool hasCabledChannels = false;
    for (unsigned int feUnitCh = 0; feUnitCh < sistrip::FEDCH_PER_FEUNIT; feUnitCh++) {
      if (connected_[iFE * sistrip::FEDCH_PER_FEUNIT + feUnitCh]) {
        hasCabledChannels = true;
        break;
      }
    }

    if (!hasCabledChannels)
      continue;

    if (aBuffer.feOverflow(iFE)) {
      lFeErr.Overflow = true;
      foundOverflow = true;
      addBadFE(lFeErr);
      //if FE overflowed then address isn't valid
      continue;
    }
    if (!aBuffer.feEnabled(iFE))
      continue;

    //check for missing data
    if (!aBuffer.fePresent(iFE)) {
      //if (hasCabledChannels) {
      lFeErr.Missing = true;
      foundMissing = true;
      addBadFE(lFeErr);
      //}
      continue;
    }
    //two independent checks for the majority address of a FE:
    //first is done inside the FED,
    //second is comparing explicitely the FE majAddress with the APVe address.
    //want to do it only for this FE... do it directly with the time difference.
    if (aBuffer.majorityAddressErrorForFEUnit(iFE)) {
      lFeErr.BadMajorityAddress = true;
      foundBadMajority = true;
      //no continue to fill the timeDifference.
    }

    //need fullDebugHeader to fill histo with time difference between APVe and FEmajAddress
    const sistrip::FEDFEHeader* header = aBuffer.feHeader();
    const sistrip::FEDFullDebugHeader* debugHeader = dynamic_cast<const sistrip::FEDFullDebugHeader*>(header);
    // if (debugHeader) {
    //   unsigned int apveTime = static_cast<unsigned int>(sistrip::FEDAddressConversion::timeLocation(aBuffer.apveAddress()));
    //   unsigned int feTime = static_cast<unsigned int>(sistrip::FEDAddressConversion::timeLocation(debugHeader->feUnitMajorityAddress(iFE)));
    //   if ((apveTime == 200 && aBuffer.apveAddress()) || feTime == 200) {
    //  std::cout << "FED " << fedID_ << ", iFE = " << iFE << std::endl
    //      << " -- aBuffer.apveAddress() = " << static_cast<unsigned int>(aBuffer.apveAddress())
    // //       << ", debugHeader = " << debugHeader
    // //       << ", header->feGood(iFE) = " << aBuffer.feGood(iFE)
    //          << ", debugHeader->feUnitMajorityAddress(iFE) " << static_cast<unsigned int>(debugHeader->feUnitMajorityAddress(iFE))
    //          << std::endl
    //          << " -- timeLoc(feUnitMajAddr) = "
    //          << static_cast<unsigned int>(sistrip::FEDAddressConversion::timeLocation(debugHeader->feUnitMajorityAddress(iFE)))
    //          << ", timeLoc(apveAddr) = "
    //          << static_cast<unsigned int>(sistrip::FEDAddressConversion::timeLocation(aBuffer.apveAddress()))
    // //       << ", aBuffer.checkFEUnitAPVAddresses() = "
    // //       << aBuffer.checkFEUnitAPVAddresses()
    //          << std::endl;
    //  std::cout << "My checks = "
    //      << ", feOverflows = " << lFeErr.Overflow << " " << foundOverflow
    //      << ", feMissing = " << lFeErr.Missing << " " << foundMissing
    //      << ", feBadMajAddr = " << lFeErr.BadMajorityAddress  << " " << foundBadMajority
    //      << std::endl;

    //  std::cout << "TimeDiff = " << feTime-apveTime << std::endl;

    //  //   std::cout << "aBuffer.checkFEUnitAPVAddresses() = " << aBuffer.checkFEUnitAPVAddresses() << std::endl;
    //   }
    // }

    lFeErr.Apve = aBuffer.apveAddress();

    if (debugHeader) {
      lFeErr.FeMaj = debugHeader->feUnitMajorityAddress(iFE);

      if (aDoFEMaj) {
        if (lFeErr.SubDetID == 2 || lFeErr.SubDetID == 3 || lFeErr.SubDetID == 4)
          aFeMajFrac[0].push_back(std::pair<unsigned int, unsigned int>(fedID_, lFeErr.FeMaj));
        else if (lFeErr.SubDetID == 5)
          aFeMajFrac[1].push_back(std::pair<unsigned int, unsigned int>(fedID_, lFeErr.FeMaj));
        else if (lFeErr.SubDetID == 0)
          aFeMajFrac[2].push_back(std::pair<unsigned int, unsigned int>(fedID_, lFeErr.FeMaj));
        else if (lFeErr.SubDetID == 1)
          aFeMajFrac[3].push_back(std::pair<unsigned int, unsigned int>(fedID_, lFeErr.FeMaj));
      }

      if (aBuffer.apveAddress()) {
        lFeErr.TimeDifference =  //0;
            static_cast<unsigned int>(
                sistrip::FEDAddressConversion::timeLocation(debugHeader->feUnitMajorityAddress(iFE))) -
            static_cast<unsigned int>(sistrip::FEDAddressConversion::timeLocation(aBuffer.apveAddress()));
        //aBuffer.apveAddress(), debugHeader->feUnitMajorityAddress(iFE)
        //FEDAddressConversion::timeLocation(const uint8_t aPipelineAddress)
      }
    }

    if (foundBadMajority || lFeErr.TimeDifference != 0) {
      addBadFE(lFeErr);

      //      LogDebug("SiStripMonitorHardware") << " -- Problem found with FE maj address :" << std::endl
      //                     << " --- FED = " << fedID_
      //                     << ", iFE = " << iFE << std::endl
      //                     << " --- aBuffer.apveAddress() = " << static_cast<unsigned int>(aBuffer.apveAddress())
      //                     << std::endl
      //                     << " --- debugHeader->feUnitMajorityAddress(iFE) " << static_cast<unsigned int>(debugHeader->feUnitMajorityAddress(iFE))<< std::endl
      //                     << " --- timeLoc(feUnitMajAddr) = "
      //                     << static_cast<unsigned int>(sistrip::FEDAddressConversion::timeLocation(debugHeader->feUnitMajorityAddress(iFE)))<< std::endl
      //                     << " --- timeLoc(apveAddr) = "
      //                     << static_cast<unsigned int>(sistrip::FEDAddressConversion::timeLocation(aBuffer.apveAddress())) << std::endl
      //                     << " --- aBuffer.checkFEUnitAPVAddresses() = "
      //                     << aBuffer.checkFEUnitAPVAddresses()
      //                     << std::endl;
    }
  }

  return !(foundOverflow || foundMissing || foundBadMajority);
}

bool FEDErrors::fillChannelErrors(const sistrip::FEDBuffer& aBuffer,
                                  bool& aFullDebug,
                                  const unsigned int aPrintDebug,
                                  unsigned int& aCounterMonitoring,
                                  unsigned int& aCounterUnpacker,
                                  const bool aDoMeds,
                                  MonitorElement* aMedianHist0,
                                  MonitorElement* aMedianHist1) {
  bool foundError = false;

  const sistrip::FEDFEHeader* header = aBuffer.feHeader();
  const sistrip::FEDFullDebugHeader* debugHeader = dynamic_cast<const sistrip::FEDFullDebugHeader*>(header);

  aFullDebug = debugHeader;

  bool lMedValid = aBuffer.readoutMode() == sistrip::READOUT_MODE_ZERO_SUPPRESSED;

  //this method is not called if there was anyFEDerrors(),
  //so only corruptBuffer+FE check are useful.
  bool lPassedMonitoringFEDcheck = !fedErrors_.CorruptBuffer;

  for (unsigned int iCh = 0; iCh < sistrip::FEDCH_PER_FED; iCh++) {  //loop on channels

    bool lFailUnpackerChannelCheck = (!aBuffer.channelGood(iCh, true) && connected_[iCh]) || failUnpackerFEDCheck_;
    bool lFailMonitoringChannelCheck = !lPassedMonitoringFEDcheck && connected_[iCh];

    FEDErrors::ChannelLevelErrors lChErr;
    lChErr.ChannelID = iCh;
    lChErr.Connected = connected_[iCh];
    lChErr.IsActive = false;
    lChErr.Unlocked = false;
    lChErr.OutOfSync = false;

    if (!connected_[iCh]) {
      //to fill histo with unconnected channels
      addBadChannel(lChErr);
      foundError = true;
    } else {  //if channel connected
      if (!aBuffer.feGood(static_cast<unsigned int>(iCh / sistrip::FEDCH_PER_FEUNIT))) {
        lFailMonitoringChannelCheck = true;
        foundError = true;
      } else {  //if FE good

        bool apvBad[2] = {false, false};
        sistrip::FEDChannelStatus lStatus = sistrip::CHANNEL_STATUS_NO_PROBLEMS;
        //CHANNEL_STATUS_NO_PROBLEMS
        //CHANNEL_STATUS_LOCKED
        //CHANNEL_STATUS_IN_SYNC
        //CHANNEL_STATUS_APV1_ADDRESS_GOOD
        //CHANNEL_STATUS_APV1_NO_ERROR_BIT
        //CHANNEL_STATUS_APV0_ADDRESS_GOOD
        //CHANNEL_STATUS_APV0_NO_ERROR_BIT

        if (debugHeader) {
          lStatus = debugHeader->getChannelStatus(iCh);
          apvBad[0] = !(lStatus & sistrip::CHANNEL_STATUS_LOCKED) || !(lStatus & sistrip::CHANNEL_STATUS_IN_SYNC) ||
                      !(lStatus & sistrip::CHANNEL_STATUS_APV0_ADDRESS_GOOD) ||
                      !(lStatus & sistrip::CHANNEL_STATUS_APV0_NO_ERROR_BIT);
          apvBad[1] = !(lStatus & sistrip::CHANNEL_STATUS_LOCKED) || !(lStatus & sistrip::CHANNEL_STATUS_IN_SYNC) ||
                      !(lStatus & sistrip::CHANNEL_STATUS_APV1_ADDRESS_GOOD) ||
                      !(lStatus & sistrip::CHANNEL_STATUS_APV1_NO_ERROR_BIT);
          //if (!debugHeader->unlocked(iCh)) {
          if (lStatus & sistrip::CHANNEL_STATUS_LOCKED) {
            lChErr.IsActive = true;
            if (lStatus == sistrip::CHANNEL_STATUS_NO_PROBLEMS)
              continue;
            //if (debugHeader->outOfSyncFromBit(iCh)) {
            if (!(lStatus & sistrip::CHANNEL_STATUS_IN_SYNC)) {
              lChErr.OutOfSync = true;
            }
          } else {
            lChErr.Unlocked = true;
          }
        } else {
          //if (header->checkChannelStatusBits(iCh)) activeChannel = true;
          apvBad[0] = !header->checkStatusBits(iCh, 0);
          apvBad[1] = !header->checkStatusBits(iCh, 1);
          if (!apvBad[0] && !apvBad[1]) {
            lChErr.IsActive = true;
            continue;
          }
        }

        if (lChErr.Unlocked || lChErr.OutOfSync)
          addBadChannel(lChErr);

        //std::ostringstream lMode;
        //lMode << aBuffer.readoutMode();

        bool lFirst = true;

        for (unsigned int iAPV = 0; iAPV < 2; iAPV++) {  //loop on APVs

          FEDErrors::APVLevelErrors lAPVErr;
          lAPVErr.APVID = 2 * iCh + iAPV;
          lAPVErr.ChannelID = iCh;
          lAPVErr.Connected = connected_[iCh];
          lAPVErr.IsActive = lChErr.IsActive;
          lAPVErr.APVStatusBit = false;
          lAPVErr.APVError = false;
          lAPVErr.APVAddressError = false;

          //if (!header->checkStatusBits(iCh,iAPV)){
          if (apvBad[iAPV]) {
            lFailMonitoringChannelCheck = true;
            lAPVErr.APVStatusBit = true;
            foundError = true;
          }

          if (debugHeader && !lChErr.Unlocked && !lChErr.OutOfSync) {
            //if (debugHeader->apvErrorFromBit(iCh,iAPV)) {
            if ((iAPV == 0 && !(lStatus & sistrip::CHANNEL_STATUS_APV0_NO_ERROR_BIT)) ||
                (iAPV == 1 && !(lStatus & sistrip::CHANNEL_STATUS_APV1_NO_ERROR_BIT))) {
              lAPVErr.APVError = true;
            }
            //if (debugHeader->apvAddressErrorFromBit(iCh,iAPV)) {
            if ((iAPV == 0 && !(lStatus & sistrip::CHANNEL_STATUS_APV0_ADDRESS_GOOD)) ||
                (iAPV == 1 && !(lStatus & sistrip::CHANNEL_STATUS_APV1_ADDRESS_GOOD))) {
              lAPVErr.APVAddressError = true;
            }
          }

          if (lAPVErr.APVStatusBit || lAPVErr.APVError || lAPVErr.APVAddressError)
            addBadAPV(lAPVErr, lFirst);
        }  //loop on APVs

      }  //if FE good
    }    //if connected

    if (lFailUnpackerChannelCheck != lFailMonitoringChannelCheck && connected_[iCh]) {
      if (aPrintDebug > 1) {
        std::ostringstream debugStream;
        debugStream << "[FEDErrors] ------ WARNING: FED " << fedID_ << ", channel " << iCh
                    << ", isConnected = " << connected_[iCh] << std::endl
                    << "[FEDErrors] --------- Monitoring Channel check ";
        if (lFailMonitoringChannelCheck)
          debugStream << "failed." << std::endl;
        else
          debugStream << "passed." << std::endl;
        debugStream << "[FEDErrors] --------- Unpacker Channel check ";
        if (lFailUnpackerChannelCheck)
          debugStream << "failed." << std::endl;
        else
          debugStream << "passed." << std::endl;
        debugStream << "[FEDErrors] --------- fegood = "
                    << aBuffer.feGood(static_cast<unsigned int>(iCh / sistrip::FEDCH_PER_FEUNIT)) << std::endl
                    << "[FEDErrors] --------- unpacker FED check = " << failUnpackerFEDCheck_ << std::endl;
        edm::LogError("SiStripMonitorHardware") << debugStream.str();
      }

      if (lFailMonitoringChannelCheck)
        aCounterMonitoring++;
      if (lFailUnpackerChannelCheck)
        aCounterUnpacker++;
    }

    if (lMedValid && !foundError && lPassedMonitoringFEDcheck && aDoMeds) {
      //get CM values
      const sistrip::FEDChannel& lChannel = aBuffer.channel(iCh);

      HistogramBase::fillHistogram(aMedianHist0, lChannel.cmMedian(0));
      HistogramBase::fillHistogram(aMedianHist1, lChannel.cmMedian(1));
    }

  }  //loop on channels

  return !foundError;
}

void FEDErrors::fillBadChannelList(const bool doTkHistoMap,
                                   TkHistoMap* aTkMapPointer,
                                   MonitorElement* aFedIdVsApvId,
                                   unsigned int& aNBadChannels,
                                   unsigned int& aNBadActiveChannels,
                                   unsigned int& aNBadChannels_perFEDID,
                                   std::vector<unsigned int>& nTotal,
                                   std::vector<unsigned int>& nErrors) {
  uint32_t lPrevId = 0;
  uint16_t nBad = 0;
  uint16_t lPrevTot = 0;
  bool hasBeenProcessed = false;
  bool lFailFED = failMonitoringFEDCheck();

  for (unsigned int iCh = 0; iCh < sistrip::FEDCH_PER_FED; iCh++) {  //loop on channels

    if (!connected_[iCh])
      continue;
    if (!detid_[iCh] || detid_[iCh] == sistrip::invalid32_)
      continue;

    if (lPrevId == 0) {
      lPrevId = detid_[iCh];
      lPrevTot = nChInModule_[iCh];
    }

    unsigned int feNumber = static_cast<unsigned int>(iCh / sistrip::FEDCH_PER_FEUNIT);

    bool isBadFE = false;
    bool isMissingFE = false;
    //feErrors vector of FE 0 - 7, each FE has channels 0 -11, 12 .. - ,... - 96
    for (unsigned int badfe(0); badfe < feErrors_.size(); badfe++) {
      if ((feErrors_[badfe]).FeID == feNumber) {
        isBadFE = true;
        if ((feErrors_[badfe]).Missing)
          isMissingFE = true;
        break;
      }
    }

    bool isBadChan = false;
    bool isActiveChan = false;
    //FED errors, apv
    //for (unsigned int badCh(0); badCh<chErrors_.size(); badCh++) {
    //if (chErrors_[badCh].first == iCh) {
    //if (chErrors_[badCh].second) isActiveChan = true;
    //isBadChan = true;
    //break;
    //}
    //}

    //apvErrors_
    bool isBadApv1 = false;
    bool isBadApv2 = false;
    for (unsigned int badApv(0); badApv < apvErrors_.size(); badApv++) {
      if ((apvErrors_[badApv]).ChannelID == iCh) {
        isBadChan = true;
        if (apvErrors_[badApv].IsActive)
          isActiveChan = true;
      }
      if (apvErrors_[badApv].APVID == 2 * iCh)
        isBadApv1 = true;
      if (apvErrors_[badApv].APVID == 2 * iCh + 1) {
        isBadApv2 = true;
        break;
      }
    }

    if (detid_[iCh] == lPrevId) {
      if (hasBeenProcessed)
        hasBeenProcessed = false;
    }
    //fill vector for previous detid
    if (detid_[iCh] != lPrevId) {
      processDet(lPrevId, lPrevTot, doTkHistoMap, nBad, aTkMapPointer);
      lPrevId = detid_[iCh];
      lPrevTot = nChInModule_[iCh];
      hasBeenProcessed = true;
    }

    bool lHasErr = lFailFED || isBadFE || isBadChan;
    incrementLumiErrors(lHasErr, subDetId_[feNumber], nTotal, nErrors);

    if (lHasErr) {
      nBad++;
      aNBadChannels++;
      aNBadChannels_perFEDID = aNBadChannels_perFEDID + 1;
      //define as active channel if channel locked AND not from an unlocked FE.
      if ((isBadChan && isActiveChan) || lFailFED || (isBadFE && !isMissingFE))
        aNBadActiveChannels++;
      if (isBadApv1 || lFailFED || isBadFE)
        HistogramBase::fillHistogram(aFedIdVsApvId, 2 * iCh, fedID_);
      if (isBadApv2 || lFailFED || isBadFE)
        HistogramBase::fillHistogram(aFedIdVsApvId, 2 * iCh + 1, fedID_);
    }

  }  //loop on channels

  if (!hasBeenProcessed) {
    processDet(lPrevId, lPrevTot, doTkHistoMap, nBad, aTkMapPointer);
  }
}

void FEDErrors::fillEventProperties(long long dbx) { eventProp_.deltaBX = dbx; }

void FEDErrors::incrementLumiErrors(const bool hasError,
                                    const unsigned int aSubDet,
                                    std::vector<unsigned int>& nTotal,
                                    std::vector<unsigned int>& nErrors) {
  if (nTotal.empty())
    return;
  if (aSubDet >= nTotal.size()) {
    edm::LogError("SiStripMonitorHardware") << " -- FED " << fedID_ << ", invalid subdetid : " << aSubDet
                                            << ", size of lumiErr : " << nTotal.size() << std::endl;
  } else {
    if (hasError)
      nErrors[aSubDet]++;
    nTotal[aSubDet]++;
  }
}

void FEDErrors::processDet(const uint32_t aPrevId,
                           const uint16_t aPrevTot,
                           const bool doTkHistoMap,
                           uint16_t& nBad,
                           TkHistoMap* aTkMapPointer) {
  if (aPrevTot < nBad) {
    edm::LogError("SiStripMonitorHardware") << " -- Number of bad channels in det " << aPrevId << " = " << nBad
                                            << ", total number of pairs for this det = " << aPrevTot << std::endl;
  }

  //tkHistoMap takes a uint & as argument
  uint32_t lDetid = aPrevId;
  if (aPrevTot != 0 && doTkHistoMap && aTkMapPointer)
    HistogramBase::fillTkHistoMap(aTkMapPointer, lDetid, static_cast<float>(nBad) / aPrevTot);

  nBad = 0;
}

const bool FEDErrors::failMonitoringFEDCheck() { return (anyFEDErrors() || fedErrors_.CorruptBuffer); }

const bool FEDErrors::anyDAQProblems() {
  return (fedErrors_.DataMissing || fedErrors_.InvalidBuffers || fedErrors_.BadFEDCRCs || fedErrors_.BadDAQCRCs ||
          fedErrors_.BadIDs || fedErrors_.BadDAQPacket);
}

const bool FEDErrors::anyFEDErrors() {
  return (fedErrors_.InvalidBuffers || fedErrors_.BadFEDCRCs || fedErrors_.BadDAQCRCs || fedErrors_.BadIDs ||
          fedErrors_.BadDAQPacket || fedErrors_.FEsOverflow);
}

const bool FEDErrors::anyFEProblems() {
  return (fedErrors_.FEsOverflow || fedErrors_.FEsMissing || fedErrors_.FEsBadMajorityAddress);
}

const bool FEDErrors::printDebug() {
  return (anyFEDErrors() || anyFEProblems() || fedErrors_.CorruptBuffer || fedErrors_.BadChannelStatusBit);
}

const unsigned int FEDErrors::fedID() { return fedID_; }

FEDErrors::FEDCounters& FEDErrors::getFEDErrorsCounters() { return lFedCounter_; }

FEDErrors::ChannelCounters& FEDErrors::getChannelErrorsCounters() { return lChCounter_; }

FEDErrors::FECounters& FEDErrors::getFEErrorsCounters() { return feCounter_; }

FEDErrors::FEDLevelErrors& FEDErrors::getFEDLevelErrors() { return fedErrors_; }

FEDErrors::EventProperties& FEDErrors::getEventProperties() { return eventProp_; }

std::vector<FEDErrors::FELevelErrors>& FEDErrors::getFELevelErrors() { return feErrors_; }

std::vector<FEDErrors::ChannelLevelErrors>& FEDErrors::getChannelLevelErrors() { return chErrorsDetailed_; }

std::vector<FEDErrors::APVLevelErrors>& FEDErrors::getAPVLevelErrors() { return apvErrors_; }

std::vector<std::pair<unsigned int, bool> >& FEDErrors::getBadChannels() { return chErrors_; }

void FEDErrors::addBadFE(const FEDErrors::FELevelErrors& aFE) {
  if (aFE.Overflow) {
    fedErrors_.FEsOverflow = true;
    (feCounter_.nFEOverflows)++;
  } else if (aFE.Missing) {
    fedErrors_.FEsMissing = true;
    (feCounter_.nFEMissing)++;
    feErrors_.push_back(aFE);
  } else if (aFE.BadMajorityAddress) {
    fedErrors_.FEsBadMajorityAddress = true;
    (feCounter_.nFEBadMajorityAddresses)++;
    feErrors_.push_back(aFE);
  } else if (aFE.TimeDifference != 0) {
    feErrors_.push_back(aFE);
  }
}

void FEDErrors::addBadChannel(const FEDErrors::ChannelLevelErrors& aChannel) {
  if (aChannel.Connected)
    chErrorsDetailed_.push_back(aChannel);
  incrementChannelCounters(aChannel);
}

void FEDErrors::addBadAPV(const FEDErrors::APVLevelErrors& aAPV, bool& aFirst) {
  apvErrors_.push_back(aAPV);
  incrementAPVCounters(aAPV);
  if (aAPV.APVStatusBit && aFirst) {
    fedErrors_.BadChannelStatusBit = true;
    lFedCounter_.nBadChannels++;
    chErrors_.push_back(std::pair<unsigned int, bool>(aAPV.ChannelID, aAPV.IsActive));
    if (aAPV.IsActive) {
      //print(aAPV);
      fedErrors_.BadActiveChannelStatusBit = true;
      lFedCounter_.nBadActiveChannels++;
      //std::cout << "------ nBadActiveChannels = " << FEDErrors::getFEDErrorsCounters().nBadActiveChannels << std::endl;
    }
    aFirst = false;
  }
}

void FEDErrors::incrementFEDCounters() {
  if (fedErrors_.InvalidBuffers || fedErrors_.BadFEDCRCs || fedErrors_.BadDAQCRCs || fedErrors_.BadIDs ||
      fedErrors_.BadDAQPacket) {
    lFedCounter_.nDAQProblems++;
    lFedCounter_.nFEDErrors++;
  }

  //FElevel errors
  if (fedErrors_.FEsOverflow) {
    lFedCounter_.nFEDsWithFEOverflows++;
  } else if (fedErrors_.FEsMissing) {
    lFedCounter_.nFEDsWithMissingFEs++;
  } else if (fedErrors_.FEsBadMajorityAddress) {
    lFedCounter_.nFEDsWithFEBadMajorityAddresses++;
  }

  if (fedErrors_.FEsOverflow || fedErrors_.FEsBadMajorityAddress || fedErrors_.FEsMissing) {
    lFedCounter_.nFEDsWithFEProblems++;
    lFedCounter_.nFEDErrors++;
  } else if (fedErrors_.CorruptBuffer) {
    lFedCounter_.nCorruptBuffers++;
    lFedCounter_.nFEDErrors++;
  }
}

void FEDErrors::incrementChannelCounters(const FEDErrors::ChannelLevelErrors& aChannel) {
  if (aChannel.Unlocked && aChannel.Connected)
    lChCounter_.nUnlocked++;
  if (aChannel.OutOfSync && aChannel.Connected)
    lChCounter_.nOutOfSync++;
  if (!aChannel.Connected)
    lChCounter_.nNotConnected++;
}

void FEDErrors::incrementAPVCounters(const FEDErrors::APVLevelErrors& aAPV) {
  if (aAPV.Connected && aAPV.IsActive) {
    if (aAPV.APVStatusBit)
      lChCounter_.nAPVStatusBit++;
    if (aAPV.APVAddressError)
      lChCounter_.nAPVAddressError++;
    if (aAPV.APVError)
      lChCounter_.nAPVError++;
  }
}

bool FEDErrors::ChannelLevelErrors::operator<(const FEDErrors::ChannelLevelErrors& aErr) const {
  if (this->ChannelID < aErr.ChannelID)
    return true;
  return false;
}

bool FEDErrors::APVLevelErrors::operator<(const FEDErrors::APVLevelErrors& aErr) const {
  if (this->ChannelID < aErr.ChannelID)
    return true;
  return false;
}

void FEDErrors::print(const FEDErrors::FEDCounters& aFEDCounter, std::ostream& aOs) {
  aOs << std::endl;
  aOs << "[FEDErrors]============================================" << std::endl
      << "[FEDErrors]==== Printing FEDCounters information : ====" << std::endl
      << "[FEDErrors]============================================" << std::endl
      << "[FEDErrors]======== nFEDErrors = " << aFEDCounter.nFEDErrors << std::endl
      << "[FEDErrors]======== nDAQProblems = " << aFEDCounter.nDAQProblems << std::endl
      << "[FEDErrors]======== nFEDsWithFEProblems = " << aFEDCounter.nFEDsWithFEProblems << std::endl
      << "[FEDErrors]======== nCorruptBuffers = " << aFEDCounter.nCorruptBuffers << std::endl
      << "[FEDErrors]======== nBadChannels = " << aFEDCounter.nBadChannels << std::endl
      << "[FEDErrors]======== nBadActiveChannels = " << aFEDCounter.nBadActiveChannels << std::endl
      << "[FEDErrors]======== nFEDsWithFEOverflows = " << aFEDCounter.nFEDsWithFEOverflows << std::endl
      << "[FEDErrors]======== nFEDsWithFEBadMajorityAddresses = " << aFEDCounter.nFEDsWithFEBadMajorityAddresses
      << std::endl
      << "[FEDErrors]======== nFEDsWithMissingFEs = " << aFEDCounter.nFEDsWithMissingFEs << std::endl
      << "[FEDErrors]======== nTotalBadChannels = " << aFEDCounter.nTotalBadChannels << std::endl
      << "[FEDErrors]======== nTotalBadActiveChannels = " << aFEDCounter.nTotalBadActiveChannels << std::endl
      << "[FEDErrors]============================================" << std::endl;
}

void FEDErrors::print(const FEDErrors::FECounters& aFECounter, std::ostream& aOs) {
  aOs << std::endl;
  aOs << "[FEDErrors]============================================" << std::endl
      << "[FEDErrors]==== Printing FECounters information :  ====" << std::endl
      << "[FEDErrors]============================================" << std::endl
      << "[FEDErrors]======== nFEOverflows = " << aFECounter.nFEOverflows << std::endl
      << "[FEDErrors]======== nFEBadMajorityAddresses = " << aFECounter.nFEBadMajorityAddresses << std::endl
      << "[FEDErrors]======== nFEMissing = " << aFECounter.nFEMissing << std::endl
      << "[FEDErrors]============================================" << std::endl;
}

void FEDErrors::print(const FEDErrors::FEDLevelErrors& aFEDErr, std::ostream& aOs) {
  aOs << std::endl;
  aOs << "[FEDErrors]============================================" << std::endl
      << "[FEDErrors]==== Printing FED errors information :  ====" << std::endl
      << "[FEDErrors]============================================" << std::endl
      << "[FEDErrors]======== HasCabledChannels = " << aFEDErr.HasCabledChannels << std::endl
      << "[FEDErrors]======== DataPresent = " << aFEDErr.DataPresent << std::endl
      << "[FEDErrors]======== DataMissing = " << aFEDErr.DataMissing << std::endl
      << "[FEDErrors]======== InvalidBuffers = " << aFEDErr.InvalidBuffers << std::endl
      << "[FEDErrors]======== BadFEDCRCs = " << aFEDErr.BadFEDCRCs << std::endl
      << "[FEDErrors]======== BadDAQCRCs = " << aFEDErr.BadDAQCRCs << std::endl
      << "[FEDErrors]======== BadIDs = " << aFEDErr.BadIDs << std::endl
      << "[FEDErrors]======== BadDAQPacket = " << aFEDErr.BadDAQPacket << std::endl
      << "[FEDErrors]======== CorruptBuffer = " << aFEDErr.CorruptBuffer << std::endl
      << "[FEDErrors]======== FEOverflows = " << aFEDErr.FEsOverflow << std::endl
      << "[FEDErrors]======== FEMissing = " << aFEDErr.FEsMissing << std::endl
      << "[FEDErrors]======== BadMajorityAddresses = " << aFEDErr.FEsBadMajorityAddress << std::endl
      << "[FEDErrors]============================================" << std::endl;
}

void FEDErrors::print(const FEDErrors::FELevelErrors& aErr, std::ostream& aOs) {
  aOs << std::endl;
  aOs << "[FEDErrors]============================================" << std::endl
      << "[FEDErrors]==== Printing FE errors information :   ====" << std::endl
      << "[FEDErrors]============================================" << std::endl
      << "[FEDErrors]======== FE #" << aErr.FeID << std::endl
      << "[FEDErrors]======== subdet " << aErr.SubDetID << std::endl
      << "[FEDErrors]======== FEOverflow = " << aErr.Overflow << std::endl
      << "[FEDErrors]======== FEMissing = " << aErr.Missing << std::endl
      << "[FEDErrors]======== BadMajorityAddresses = " << aErr.BadMajorityAddress << std::endl
      << "[FEDErrors]======== TimeDifference = " << aErr.TimeDifference << std::endl
      << "[FEDErrors]============================================" << std::endl;
}

void FEDErrors::print(const FEDErrors::ChannelLevelErrors& aErr, std::ostream& aOs) {
  aOs << std::endl;
  aOs << "[FEDErrors]=================================================" << std::endl
      << "[FEDErrors]==== Printing channel errors information :   ====" << std::endl
      << "[FEDErrors]=================================================" << std::endl
      << "[FEDErrors]============ Channel #" << aErr.ChannelID << std::endl
      << "[FEDErrors]============ connected  = " << aErr.Connected << std::endl
      << "[FEDErrors]============ isActive  = " << aErr.IsActive << std::endl
      << "[FEDErrors]============ Unlocked = " << aErr.Unlocked << std::endl
      << "[FEDErrors]============ OutOfSync = " << aErr.OutOfSync << std::endl
      << "[FEDErrors]=================================================" << std::endl;
}

void FEDErrors::print(const FEDErrors::APVLevelErrors& aErr, std::ostream& aOs) {
  aOs << std::endl;
  aOs << "[FEDErrors]=================================================" << std::endl
      << "[FEDErrors]==== Printing APV errors information :       ====" << std::endl
      << "[FEDErrors]=================================================" << std::endl
      << "[FEDErrors]============ APV #" << aErr.APVID << std::endl
      << "[FEDErrors]============ Channel #" << aErr.ChannelID << std::endl
      << "[FEDErrors]============ connected  = " << aErr.Connected << std::endl
      << "[FEDErrors]============ isActive  = " << aErr.IsActive << std::endl
      << "[FEDErrors]============ APVStatusBit = " << aErr.APVStatusBit << std::endl
      << "[FEDErrors]============ APVError = " << aErr.APVError << std::endl
      << "[FEDErrors]============ APVAddressError = " << aErr.APVAddressError << std::endl
      << "[FEDErrors]=================================================" << std::endl;
}