InputFile Class Reference

#include <FedRawDataInputSource.h>

Inheritance diagram for InputFile:

Public Member Functions
bool	advance (unsigned char *&dataPosition, const size_t size)
void	appendFile (std::string const &name, uint64_t size)
uint64_t	currentChunkSize () const
int64_t	fileSizeLeft () const
	InputFile (evf::EvFDaqDirector::FileStatus status, unsigned int lumi=0, std::string const &name=std::string(), bool deleteFile=true, int rawFd=-1, uint64_t fileSize=0, uint16_t rawHeaderSize=0, uint16_t nChunks=0, int nEvents=0, FedRawDataInputSource *parent=nullptr)
void	moveToPreviousChunk (const size_t size, const size_t offset)
void	randomizeOrder (std::default_random_engine &rng)
void	rewindChunk (const size_t size)
void	setChunks (uint16_t nChunks)
void	unsetDeleteFile ()
bool	waitForChunk (unsigned int chunkid)
virtual	~InputFile ()

Public Attributes
std::vector< uint64_t >	bufferOffsets_
uint32_t	bufferPosition_ = 0
uint32_t	chunkPosition_ = 0
tbb::concurrent_vector< InputChunk * >	chunks_
unsigned int	currentChunk_ = 0
bool	deleteFile_
std::vector< uint64_t >	diskFileSizes_
std::string	fileName_
std::vector< std::string >	fileNames_
std::vector< unsigned int >	fileOrder_
uint64_t	fileSize_
std::vector< uint64_t >	fileSizes_
unsigned int	lumi_
uint16_t	nChunks_
int	nEvents_
unsigned int	nProcessed_
uint16_t	numFiles_
FedRawDataInputSource *	parent_
int	rawFd_
uint16_t	rawHeaderSize_
evf::EvFDaqDirector::FileStatus	status_

Detailed Description

Definition at line 223 of file FedRawDataInputSource.h.

Constructor & Destructor Documentation

InputFile()

InputFile::InputFile ( evf::EvFDaqDirector::FileStatus  status, unsigned int  lumi = 0, std::string const &  name = std::string(), bool  deleteFile = true, int  rawFd = -1, uint64_t  fileSize = 0, uint16_t  rawHeaderSize = 0, uint16_t  nChunks = 0, int  nEvents = 0, FedRawDataInputSource *  parent = nullptr  )

inline

Definition at line 250 of file FedRawDataInputSource.h.

References bufferOffsets_, chunks_, diskFileSizes_, fileNames_, fileOrder_, fileSizes_, mps_fire::i, Skims_PA_cff::name, and nChunks_.

      : parent_(parent),
        status_(status),
        lumi_(lumi),
        fileName_(name),
        deleteFile_(deleteFile),
        rawFd_(rawFd),
        fileSize_(fileSize),
        rawHeaderSize_(rawHeaderSize),
        nChunks_(nChunks),
        numFiles_(1),
        nEvents_(nEvents),
        nProcessed_(0) {
    fileNames_.push_back(name);
    fileOrder_.push_back(fileOrder_.size());
    diskFileSizes_.push_back(fileSize);
    fileSizes_.push_back(0);
    bufferOffsets_.push_back(0);
    chunks_.reserve(nChunks_);
    for (unsigned int i = 0; i < nChunks; i++)
      chunks_.push_back(nullptr);
  }

~InputFile()

InputFile::~InputFile ( )

virtual

Definition at line 1491 of file FedRawDataInputSource.cc.

References deleteFile_, cppFunctionSkipper::exception, MillePedeFileConverter_cfg::fileName, fileNames_, dt4ml_dqm_sourceclient-live_cfg::filePath, LogDebug, castor_dqm_sourceclient_file_cfg::path, rawFd_, and MatrixUtil::remove().

                      {
  if (rawFd_ != -1)
    close(rawFd_);

  if (deleteFile_) {
    for (auto& fileName : fileNames_) {
      if (!fileName.empty()) {
        const std::filesystem::path filePath(fileName);
        try {
          //sometimes this fails but file gets deleted
          LogDebug("FedRawDataInputSource:InputFile") << "Deleting input file -:" << fileName;
          std::filesystem::remove(filePath);
          continue;
        } catch (const std::filesystem::filesystem_error& ex) {
          edm::LogError("FedRawDataInputSource:InputFile")
              << " - deleteFile BOOST FILESYSTEM ERROR CAUGHT -: " << ex.what() << ". Trying again.";
        } catch (std::exception& ex) {
          edm::LogError("FedRawDataInputSource:InputFile")
              << " - deleteFile std::exception CAUGHT -: " << ex.what() << ". Trying again.";
        }
        std::filesystem::remove(filePath);
      }
    }
  }
}

Member Function Documentation

advance()

bool InputFile::advance ( unsigned char *&  dataPosition, const size_t  size  )

inline

Definition at line 1436 of file FedRawDataInputSource.cc.

References cms::cuda::assert(), bufferPosition_, chunkPosition_, chunks_, currentChunk_, FedRawDataInputSource::exceptionState(), evf::FastMonState::inChunkReceived, evf::FastMonState::inWaitChunk, parent_, FedRawDataInputSource::setMonState(), findQualityFiles::size, FedRawDataInputSource::threadError(), and waitForChunk().

                                                                              {
  //wait for chunk

  while (!waitForChunk(currentChunk_)) {
    parent_->setMonState(inWaitChunk);
    usleep(100000);
    parent_->setMonState(inChunkReceived);
    if (parent_->exceptionState())
      parent_->threadError();
  }

  dataPosition = chunks_[currentChunk_]->buf_ + chunkPosition_;
  size_t currentLeft = chunks_[currentChunk_]->size_ - chunkPosition_;

  if (currentLeft < size) {
    //we need next chunk
    assert(chunks_.size() > currentChunk_ + 1);
    while (!waitForChunk(currentChunk_ + 1)) {
      parent_->setMonState(inWaitChunk);
      usleep(100000);
      parent_->setMonState(inChunkReceived);
      if (parent_->exceptionState())
        parent_->threadError();
    }
    //copy everything to beginning of the first chunk
    dataPosition -= chunkPosition_;
    assert(dataPosition == chunks_[currentChunk_]->buf_);
    memmove(chunks_[currentChunk_]->buf_, chunks_[currentChunk_]->buf_ + chunkPosition_, currentLeft);
    memcpy(chunks_[currentChunk_]->buf_ + currentLeft, chunks_[currentChunk_ + 1]->buf_, size - currentLeft);
    //set pointers at the end of the old data position
    bufferPosition_ += size;
    chunkPosition_ = size - currentLeft;
    currentChunk_++;
    return true;
  } else {
    chunkPosition_ += size;
    bufferPosition_ += size;
    return false;
  }
}

appendFile()

void InputFile::appendFile ( std::string const &  name, uint64_t  size  )

inline

Definition at line 291 of file FedRawDataInputSource.h.

References bufferOffsets_, diskFileSizes_, fileNames_, fileOrder_, fileSizes_, Skims_PA_cff::name, numFiles_, and findQualityFiles::size.

                                                        {
    size_t prevOffset = bufferOffsets_.back();
    size_t prevSize = diskFileSizes_.back();
    numFiles_++;
    fileNames_.push_back(name);
    fileOrder_.push_back(fileOrder_.size());
    diskFileSizes_.push_back(size);
    fileSizes_.push_back(0);
    bufferOffsets_.push_back(prevOffset + prevSize);
  }

currentChunkSize()

uint64_t InputFile::currentChunkSize ( ) const

inline

Definition at line 313 of file FedRawDataInputSource.h.

References chunks_, and currentChunk_.

{ return chunks_[currentChunk_]->size_; }

fileSizeLeft()

int64_t InputFile::fileSizeLeft ( ) const

inline

Definition at line 314 of file FedRawDataInputSource.h.

References bufferPosition_, and fileSize_.

{ return (int64_t)fileSize_ - (int64_t)bufferPosition_; }

moveToPreviousChunk()

void InputFile::moveToPreviousChunk	(	const size_t	size,
		const size_t	offset
	)

Definition at line 1477 of file FedRawDataInputSource.cc.

References cms::cuda::assert(), bufferPosition_, chunkPosition_, chunks_, currentChunk_, hltrates_dqm_sourceclient-live_cfg::offset, and findQualityFiles::size.

                                                                          {
  //this will fail in case of events that are too large
  assert(size < chunks_[currentChunk_]->size_ - chunkPosition_);
  assert(size - offset < chunks_[currentChunk_]->size_);
  memcpy(chunks_[currentChunk_ - 1]->buf_ + offset, chunks_[currentChunk_]->buf_ + chunkPosition_, size);
  chunkPosition_ += size;
  bufferPosition_ += size;
}

randomizeOrder()

void InputFile::randomizeOrder ( std::default_random_engine &  rng )

inline

Definition at line 310 of file FedRawDataInputSource.h.

References fileOrder_, and emtf::shuffle().

                                                     {
    std::shuffle(std::begin(fileOrder_), std::end(fileOrder_), rng);
  }

rewindChunk()

void InputFile::rewindChunk

(

const size_t

size

)

Definition at line 1486 of file FedRawDataInputSource.cc.

References bufferPosition_, chunkPosition_, and findQualityFiles::size.

                                             {
  chunkPosition_ -= size;
  bufferPosition_ -= size;
}

setChunks()

void InputFile::setChunks ( uint16_t  nChunks )

inline

Definition at line 283 of file FedRawDataInputSource.h.

References chunks_, mps_fire::i, and nChunks_.

                                   {
    nChunks_ = nChunks;
    chunks_.clear();
    chunks_.reserve(nChunks_);
    for (unsigned int i = 0; i < nChunks_; i++)
      chunks_.push_back(nullptr);
  }

unsetDeleteFile()

void InputFile::unsetDeleteFile ( )

inline

Definition at line 309 of file FedRawDataInputSource.h.

References deleteFile_.

{ deleteFile_ = false; }

waitForChunk()

bool InputFile::waitForChunk ( unsigned int  chunkid )

inline

Definition at line 302 of file FedRawDataInputSource.h.

References chunks_.

Referenced by RawInputFile::advance(), and advance().

                                          {
    //some atomics to make sure everything is cache synchronized for the main thread
    return chunks_[chunkid] != nullptr && chunks_[chunkid]->readComplete_;
  }

Member Data Documentation

bufferOffsets_

std::vector<uint64_t> InputFile::bufferOffsets_

Definition at line 232 of file FedRawDataInputSource.h.

Referenced by appendFile(), and InputFile().

bufferPosition_

uint32_t InputFile::bufferPosition_ = 0

Definition at line 246 of file FedRawDataInputSource.h.

Referenced by RawInputFile::advance(), advance(), fileSizeLeft(), moveToPreviousChunk(), and rewindChunk().

chunkPosition_

uint32_t InputFile::chunkPosition_ = 0

Definition at line 247 of file FedRawDataInputSource.h.

Referenced by RawInputFile::advance(), advance(), moveToPreviousChunk(), and rewindChunk().

chunks_

tbb::concurrent_vector<InputChunk*> InputFile::chunks_

Definition at line 244 of file FedRawDataInputSource.h.

Referenced by RawInputFile::advance(), advance(), currentChunkSize(), InputFile(), moveToPreviousChunk(), setChunks(), and waitForChunk().

currentChunk_

unsigned int InputFile::currentChunk_ = 0

Definition at line 248 of file FedRawDataInputSource.h.

Referenced by RawInputFile::advance(), advance(), currentChunkSize(), and moveToPreviousChunk().

deleteFile_

bool InputFile::deleteFile_

Definition at line 235 of file FedRawDataInputSource.h.

Referenced by unsetDeleteFile(), and ~InputFile().

diskFileSizes_

std::vector<uint64_t> InputFile::diskFileSizes_

Definition at line 231 of file FedRawDataInputSource.h.

Referenced by appendFile(), and InputFile().

fileName_

std::string InputFile::fileName_

Definition at line 228 of file FedRawDataInputSource.h.

fileNames_

std::vector<std::string> InputFile::fileNames_

Definition at line 230 of file FedRawDataInputSource.h.

Referenced by appendFile(), InputFile(), and ~InputFile().

fileOrder_

std::vector<unsigned int> InputFile::fileOrder_

Definition at line 234 of file FedRawDataInputSource.h.

Referenced by appendFile(), InputFile(), and randomizeOrder().

fileSize_

uint64_t InputFile::fileSize_

Definition at line 237 of file FedRawDataInputSource.h.

Referenced by fileSizeLeft().

fileSizes_

std::vector<uint64_t> InputFile::fileSizes_

Definition at line 233 of file FedRawDataInputSource.h.

Referenced by appendFile(), and InputFile().

lumi_

unsigned int InputFile::lumi_

Definition at line 227 of file FedRawDataInputSource.h.

nChunks_

uint16_t InputFile::nChunks_

Definition at line 239 of file FedRawDataInputSource.h.

Referenced by InputFile(), and setChunks().

nEvents_

int InputFile::nEvents_

Definition at line 241 of file FedRawDataInputSource.h.

nProcessed_

unsigned int InputFile::nProcessed_

Definition at line 242 of file FedRawDataInputSource.h.

numFiles_

uint16_t InputFile::numFiles_

Definition at line 240 of file FedRawDataInputSource.h.

Referenced by appendFile().

parent_

FedRawDataInputSource* InputFile::parent_

Definition at line 225 of file FedRawDataInputSource.h.

Referenced by advance().

rawFd_

int InputFile::rawFd_

Definition at line 236 of file FedRawDataInputSource.h.

Referenced by ~InputFile().

rawHeaderSize_

uint16_t InputFile::rawHeaderSize_

Definition at line 238 of file FedRawDataInputSource.h.

status_

evf::EvFDaqDirector::FileStatus InputFile::status_

Definition at line 226 of file FedRawDataInputSource.h.