CMS 3D CMS Logo

List of all members | Public Member Functions | Public Attributes
InputFile Class Reference

#include <FedRawDataInputSource.h>

Inheritance diagram for InputFile:
RawInputFile

Public Member Functions

bool advance (unsigned char *&dataPosition, const size_t size)
 
void appendFile (std::string const &name, uint64_t size)
 
uint64_t currentChunkSize () const
 
int64_t fileSizeLeft () const
 
 InputFile (evf::EvFDaqDirector::FileStatus status, unsigned int lumi=0, std::string const &name=std::string(), bool deleteFile=true, int rawFd=-1, uint64_t fileSize=0, uint16_t rawHeaderSize=0, uint16_t nChunks=0, int nEvents=0, FedRawDataInputSource *parent=nullptr)
 
void moveToPreviousChunk (const size_t size, const size_t offset)
 
void randomizeOrder (std::default_random_engine &rng)
 
void rewindChunk (const size_t size)
 
void setChunks (uint16_t nChunks)
 
void unsetDeleteFile ()
 
bool waitForChunk (unsigned int chunkid)
 
virtual ~InputFile ()
 

Public Attributes

std::vector< uint64_t > bufferOffsets_
 
uint32_t bufferPosition_ = 0
 
uint32_t chunkPosition_ = 0
 
tbb::concurrent_vector< InputChunk * > chunks_
 
unsigned int currentChunk_ = 0
 
bool deleteFile_
 
std::vector< uint64_t > diskFileSizes_
 
std::string fileName_
 
std::vector< std::string > fileNames_
 
std::vector< unsigned int > fileOrder_
 
uint64_t fileSize_
 
std::vector< uint64_t > fileSizes_
 
unsigned int lumi_
 
uint16_t nChunks_
 
int nEvents_
 
unsigned int nProcessed_
 
uint16_t numFiles_
 
FedRawDataInputSourceparent_
 
int rawFd_
 
uint16_t rawHeaderSize_
 
evf::EvFDaqDirector::FileStatus status_
 

Detailed Description

Definition at line 223 of file FedRawDataInputSource.h.

Constructor & Destructor Documentation

◆ InputFile()

InputFile::InputFile ( evf::EvFDaqDirector::FileStatus  status,
unsigned int  lumi = 0,
std::string const &  name = std::string(),
bool  deleteFile = true,
int  rawFd = -1,
uint64_t  fileSize = 0,
uint16_t  rawHeaderSize = 0,
uint16_t  nChunks = 0,
int  nEvents = 0,
FedRawDataInputSource parent = nullptr 
)
inline

Definition at line 250 of file FedRawDataInputSource.h.

References bufferOffsets_, chunks_, diskFileSizes_, fileNames_, fileOrder_, fileSizes_, mps_fire::i, mergeVDriftHistosByStation::name, and nChunks_.

260  : parent_(parent),
261  status_(status),
262  lumi_(lumi),
263  fileName_(name),
264  deleteFile_(deleteFile),
265  rawFd_(rawFd),
266  fileSize_(fileSize),
267  rawHeaderSize_(rawHeaderSize),
268  nChunks_(nChunks),
269  numFiles_(1),
270  nEvents_(nEvents),
271  nProcessed_(0) {
272  fileNames_.push_back(name);
273  fileOrder_.push_back(fileOrder_.size());
274  diskFileSizes_.push_back(fileSize);
275  fileSizes_.push_back(0);
276  bufferOffsets_.push_back(0);
277  chunks_.reserve(nChunks_);
278  for (unsigned int i = 0; i < nChunks; i++)
279  chunks_.push_back(nullptr);
280  }
std::vector< uint64_t > fileSizes_
std::vector< std::string > fileNames_
FedRawDataInputSource * parent_
std::vector< uint64_t > bufferOffsets_
uint16_t rawHeaderSize_
evf::EvFDaqDirector::FileStatus status_
tbb::concurrent_vector< InputChunk * > chunks_
unsigned int nProcessed_
unsigned int lumi_
std::vector< unsigned int > fileOrder_
std::vector< uint64_t > diskFileSizes_
std::string fileName_

◆ ~InputFile()

InputFile::~InputFile ( )
virtual

Definition at line 1504 of file FedRawDataInputSource.cc.

References deleteFile_, cppFunctionSkipper::exception, MillePedeFileConverter_cfg::fileName, fileNames_, dt4ml_dqm_sourceclient-live_cfg::filePath, LogDebug, EnsembleCalibrationLA_cfg::path, rawFd_, and MatrixUtil::remove().

1504  {
1505  if (rawFd_ != -1)
1506  close(rawFd_);
1507 
1508  if (deleteFile_) {
1509  for (auto& fileName : fileNames_) {
1510  if (!fileName.empty()) {
1512  try {
1513  //sometimes this fails but file gets deleted
1514  LogDebug("FedRawDataInputSource:InputFile") << "Deleting input file -:" << fileName;
1516  continue;
1517  } catch (const std::filesystem::filesystem_error& ex) {
1518  edm::LogError("FedRawDataInputSource:InputFile")
1519  << " - deleteFile BOOST FILESYSTEM ERROR CAUGHT -: " << ex.what() << ". Trying again.";
1520  } catch (std::exception& ex) {
1521  edm::LogError("FedRawDataInputSource:InputFile")
1522  << " - deleteFile std::exception CAUGHT -: " << ex.what() << ". Trying again.";
1523  }
1525  }
1526  }
1527  }
1528 }
std::vector< std::string > fileNames_
Log< level::Error, false > LogError
def remove(d, key, TELL=False)
Definition: MatrixUtil.py:234
#define LogDebug(id)

Member Function Documentation

◆ advance()

bool InputFile::advance ( unsigned char *&  dataPosition,
const size_t  size 
)
inline

Definition at line 1449 of file FedRawDataInputSource.cc.

References cms::cuda::assert(), bufferPosition_, chunkPosition_, chunks_, currentChunk_, FedRawDataInputSource::exceptionState(), evf::FastMonState::inChunkReceived, evf::FastMonState::inWaitChunk, parent_, FedRawDataInputSource::setMonState(), findQualityFiles::size, FedRawDataInputSource::threadError(), and waitForChunk().

1449  {
1450  //wait for chunk
1451 
1452  while (!waitForChunk(currentChunk_)) {
1454  usleep(100000);
1456  if (parent_->exceptionState())
1457  parent_->threadError();
1458  }
1459 
1460  dataPosition = chunks_[currentChunk_]->buf_ + chunkPosition_;
1461  size_t currentLeft = chunks_[currentChunk_]->size_ - chunkPosition_;
1462 
1463  if (currentLeft < size) {
1464  //we need next chunk
1465  assert(chunks_.size() > currentChunk_ + 1);
1466  while (!waitForChunk(currentChunk_ + 1)) {
1468  usleep(100000);
1470  if (parent_->exceptionState())
1471  parent_->threadError();
1472  }
1473  //copy everything to beginning of the first chunk
1474  dataPosition -= chunkPosition_;
1475  assert(dataPosition == chunks_[currentChunk_]->buf_);
1476  memmove(chunks_[currentChunk_]->buf_, chunks_[currentChunk_]->buf_ + chunkPosition_, currentLeft);
1477  memcpy(chunks_[currentChunk_]->buf_ + currentLeft, chunks_[currentChunk_ + 1]->buf_, size - currentLeft);
1478  //set pointers at the end of the old data position
1479  bufferPosition_ += size;
1480  chunkPosition_ = size - currentLeft;
1481  currentChunk_++;
1482  return true;
1483  } else {
1484  chunkPosition_ += size;
1485  bufferPosition_ += size;
1486  return false;
1487  }
1488 }
size
Write out results.
FedRawDataInputSource * parent_
assert(be >=bs)
bool waitForChunk(unsigned int chunkid)
void setMonState(evf::FastMonState::InputState state)
tbb::concurrent_vector< InputChunk * > chunks_
uint32_t chunkPosition_
unsigned int currentChunk_
uint32_t bufferPosition_

◆ appendFile()

void InputFile::appendFile ( std::string const &  name,
uint64_t  size 
)
inline

Definition at line 291 of file FedRawDataInputSource.h.

References bufferOffsets_, diskFileSizes_, fileNames_, fileOrder_, fileSizes_, mergeVDriftHistosByStation::name, numFiles_, and findQualityFiles::size.

291  {
292  size_t prevOffset = bufferOffsets_.back();
293  size_t prevSize = diskFileSizes_.back();
294  numFiles_++;
295  fileNames_.push_back(name);
296  fileOrder_.push_back(fileOrder_.size());
297  diskFileSizes_.push_back(size);
298  fileSizes_.push_back(0);
299  bufferOffsets_.push_back(prevOffset + prevSize);
300  }
size
Write out results.
std::vector< uint64_t > fileSizes_
std::vector< std::string > fileNames_
std::vector< uint64_t > bufferOffsets_
std::vector< unsigned int > fileOrder_
std::vector< uint64_t > diskFileSizes_

◆ currentChunkSize()

uint64_t InputFile::currentChunkSize ( ) const
inline

Definition at line 313 of file FedRawDataInputSource.h.

References chunks_, and currentChunk_.

313 { return chunks_[currentChunk_]->size_; }
tbb::concurrent_vector< InputChunk * > chunks_
unsigned int currentChunk_

◆ fileSizeLeft()

int64_t InputFile::fileSizeLeft ( ) const
inline

Definition at line 314 of file FedRawDataInputSource.h.

References bufferPosition_, and fileSize_.

314 { return (int64_t)fileSize_ - (int64_t)bufferPosition_; }
uint32_t bufferPosition_

◆ moveToPreviousChunk()

void InputFile::moveToPreviousChunk ( const size_t  size,
const size_t  offset 
)

Definition at line 1490 of file FedRawDataInputSource.cc.

References cms::cuda::assert(), bufferPosition_, chunkPosition_, chunks_, currentChunk_, HLT_IsoTrack_cff::offset, and findQualityFiles::size.

1490  {
1491  //this will fail in case of events that are too large
1493  assert(size - offset < chunks_[currentChunk_]->size_);
1494  memcpy(chunks_[currentChunk_ - 1]->buf_ + offset, chunks_[currentChunk_]->buf_ + chunkPosition_, size);
1495  chunkPosition_ += size;
1496  bufferPosition_ += size;
1497 }
size
Write out results.
assert(be >=bs)
tbb::concurrent_vector< InputChunk * > chunks_
uint32_t chunkPosition_
unsigned int currentChunk_
uint32_t bufferPosition_

◆ randomizeOrder()

void InputFile::randomizeOrder ( std::default_random_engine &  rng)
inline

Definition at line 310 of file FedRawDataInputSource.h.

References fileOrder_, and emtf::shuffle().

310  {
311  std::shuffle(std::begin(fileOrder_), std::end(fileOrder_), rng);
312  }
std::vector< unsigned int > fileOrder_
bidiiter shuffle(bidiiter begin, bidiiter end, size_t num_random)
Definition: Utilities.h:27

◆ rewindChunk()

void InputFile::rewindChunk ( const size_t  size)

Definition at line 1499 of file FedRawDataInputSource.cc.

References bufferPosition_, chunkPosition_, and findQualityFiles::size.

1499  {
1500  chunkPosition_ -= size;
1501  bufferPosition_ -= size;
1502 }
size
Write out results.
uint32_t chunkPosition_
uint32_t bufferPosition_

◆ setChunks()

void InputFile::setChunks ( uint16_t  nChunks)
inline

Definition at line 283 of file FedRawDataInputSource.h.

References chunks_, mps_fire::i, and nChunks_.

283  {
284  nChunks_ = nChunks;
285  chunks_.clear();
286  chunks_.reserve(nChunks_);
287  for (unsigned int i = 0; i < nChunks_; i++)
288  chunks_.push_back(nullptr);
289  }
tbb::concurrent_vector< InputChunk * > chunks_

◆ unsetDeleteFile()

void InputFile::unsetDeleteFile ( )
inline

Definition at line 309 of file FedRawDataInputSource.h.

References deleteFile_.

309 { deleteFile_ = false; }

◆ waitForChunk()

bool InputFile::waitForChunk ( unsigned int  chunkid)
inline

Definition at line 302 of file FedRawDataInputSource.h.

References chunks_.

Referenced by RawInputFile::advance(), and advance().

302  {
303  //some atomics to make sure everything is cache synchronized for the main thread
304  return chunks_[chunkid] != nullptr && chunks_[chunkid]->readComplete_;
305  }
tbb::concurrent_vector< InputChunk * > chunks_

Member Data Documentation

◆ bufferOffsets_

std::vector<uint64_t> InputFile::bufferOffsets_

Definition at line 232 of file FedRawDataInputSource.h.

Referenced by appendFile(), and InputFile().

◆ bufferPosition_

uint32_t InputFile::bufferPosition_ = 0

◆ chunkPosition_

uint32_t InputFile::chunkPosition_ = 0

◆ chunks_

tbb::concurrent_vector<InputChunk*> InputFile::chunks_

◆ currentChunk_

unsigned int InputFile::currentChunk_ = 0

◆ deleteFile_

bool InputFile::deleteFile_

Definition at line 235 of file FedRawDataInputSource.h.

Referenced by unsetDeleteFile(), and ~InputFile().

◆ diskFileSizes_

std::vector<uint64_t> InputFile::diskFileSizes_

Definition at line 231 of file FedRawDataInputSource.h.

Referenced by appendFile(), and InputFile().

◆ fileName_

std::string InputFile::fileName_

Definition at line 228 of file FedRawDataInputSource.h.

◆ fileNames_

std::vector<std::string> InputFile::fileNames_

Definition at line 230 of file FedRawDataInputSource.h.

Referenced by appendFile(), InputFile(), and ~InputFile().

◆ fileOrder_

std::vector<unsigned int> InputFile::fileOrder_

Definition at line 234 of file FedRawDataInputSource.h.

Referenced by appendFile(), InputFile(), and randomizeOrder().

◆ fileSize_

uint64_t InputFile::fileSize_

Definition at line 237 of file FedRawDataInputSource.h.

Referenced by fileSizeLeft().

◆ fileSizes_

std::vector<uint64_t> InputFile::fileSizes_

Definition at line 233 of file FedRawDataInputSource.h.

Referenced by appendFile(), and InputFile().

◆ lumi_

unsigned int InputFile::lumi_

Definition at line 227 of file FedRawDataInputSource.h.

◆ nChunks_

uint16_t InputFile::nChunks_

Definition at line 239 of file FedRawDataInputSource.h.

Referenced by InputFile(), and setChunks().

◆ nEvents_

int InputFile::nEvents_

Definition at line 241 of file FedRawDataInputSource.h.

◆ nProcessed_

unsigned int InputFile::nProcessed_

Definition at line 242 of file FedRawDataInputSource.h.

◆ numFiles_

uint16_t InputFile::numFiles_

Definition at line 240 of file FedRawDataInputSource.h.

Referenced by appendFile().

◆ parent_

FedRawDataInputSource* InputFile::parent_

Definition at line 225 of file FedRawDataInputSource.h.

Referenced by advance().

◆ rawFd_

int InputFile::rawFd_

Definition at line 236 of file FedRawDataInputSource.h.

Referenced by ~InputFile().

◆ rawHeaderSize_

uint16_t InputFile::rawHeaderSize_

Definition at line 238 of file FedRawDataInputSource.h.

◆ status_

evf::EvFDaqDirector::FileStatus InputFile::status_

Definition at line 226 of file FedRawDataInputSource.h.