#include <FileHandler.h>

Inheritance diagram for stor::FileHandler:

Public Member Functions
virtual void	closeFile (const FilesMonitorCollection::FileRecord::ClosingReason &)=0
uint32_t	events () const
	FileHandler (FilesMonitorCollection::FileRecordPtr, const DbFileHandlerPtr, const DiskWritingParams &, const uint64_t &maxFileSize)
uint64_t	fileSize () const
bool	isFromLumiSection (const uint32_t lumiSection)
uint32_t	lumiSection () const
bool	tooLarge (const uint64_t &dataSize)
bool	tooOld (const utils::TimePoint_t currentTime=utils::getCurrentTime())
void	writeEvent (const I2OChain &)
virtual	~FileHandler ()
Protected Member Functions
virtual void	do_writeEvent (const I2OChain &event)=0
void	insertFileInDatabase () const
void	moveFileToClosed (const FilesMonitorCollection::FileRecord::ClosingReason &)
void	updateDatabase () const
Protected Attributes
const DbFileHandlerPtr	dbFileHandler_
const DiskWritingParams &	diskWritingParams_
FilesMonitorCollection::FileRecordPtr	fileRecord_
utils::TimePoint_t	firstEntry_
utils::TimePoint_t	lastEntry_
Private Member Functions
double	calcPctDiff (const uint64_t &, const uint64_t &) const
void	checkAdler32 (const std::string &fileName) const
void	checkDirectories () const
uint64_t	checkFileSizeMatch (const std::string &fileName, const uint64_t &size) const
	FileHandler (FileHandler const &)
std::string	logFile (const DiskWritingParams &) const
void	makeFileReadOnly (const std::string &fileName) const
FileHandler &	operator= (FileHandler const &)
void	renameFile (const std::string &openFileName, const std::string &closedFileName) const
bool	sizeMismatch (const uint64_t &initialSize, const uint64_t &finalSize) const
Private Attributes
std::string	cmsver_
const std::string	logFile_
const std::string	logPath_
const uint64_t	maxFileSize_

Detailed Description

Abstract representation of a physical file

Author:: mommsen

Revision:: 1.20

Date:: 2011/07/07 09:22:44

Definition at line 32 of file FileHandler.h.

Constructor & Destructor Documentation

stor::FileHandler::FileHandler	(	FilesMonitorCollection::FileRecordPtr	fileRecord,
		const DbFileHandlerPtr	dbFileHandler,
		const DiskWritingParams &	dwParams,
		const uint64_t &	maxFileSize
	)

Definition at line 26 of file FileHandler.cc.

   :
  fileRecord_(fileRecord),
  dbFileHandler_(dbFileHandler),
  firstEntry_(utils::getCurrentTime()),
  lastEntry_(firstEntry_),
  diskWritingParams_(dwParams),
  maxFileSize_(maxFileSize),
  cmsver_(edm::getReleaseVersion())
  {
    // stripp quotes if present
    if(cmsver_[0]=='"') cmsver_=cmsver_.substr(1,cmsver_.size()-2);
    
    checkDirectories();
    
    insertFileInDatabase();
  }

virtual stor::FileHandler::~FileHandler ( ) [inline, virtual]

Definition at line 44 of file FileHandler.h.

{};

stor::FileHandler::FileHandler ( FileHandler const & ) [private]

Member Function Documentation

double stor::FileHandler::calcPctDiff	(	const uint64_t &	value1,
		const uint64_t &	value2
	)		const `[private]`

Return the relative difference btw to file sizes

Definition at line 342 of file FileHandler.cc.

References max(), and min.

Referenced by sizeMismatch().

  {
    if (value1 == value2) return 0;
    uint64_t largerValue = std::max(value1,value2);
    uint64_t smallerValue = std::min(value1,value2);
    return ( largerValue > 0 ? (largerValue - smallerValue) / largerValue : 0 );
  }

void stor::FileHandler::checkAdler32 ( const std::string & fileName ) const [private]

Compare Adler32 checksum of file with calculated one

Definition at line 274 of file FileHandler.cc.

References a, cms::Adler32(), b, stor::DiskWritingParams::checkAdler32_, diskWritingParams_, mergeVDriftHistosByStation::file, fileRecord_, recoMuon::in, MatrixRunner::msg, and findQualityFiles::size.

  {
    if (!diskWritingParams_.checkAdler32_) return;

    std::ifstream file(fileName.c_str(), std::ios::in | std::ios::binary | std::ios::ate);
    if (!file.is_open())
    {
      std::ostringstream msg;
      msg << "File " << fileName << " could not be opened.\n";
      XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
    }
    
    std::ifstream::pos_type size = file.tellg();
    file.seekg (0, std::ios::beg);
    
    boost::shared_array<char> ptr(new char[1024*1024]);
    uint32_t a = 1, b = 0;
    
    std::ifstream::pos_type rsize = 0;
    while(rsize < size)
    {
      file.read(ptr.get(), 1024*1024);
      rsize += 1024*1024;
      if(file.gcount()) 
        cms::Adler32(ptr.get(), file.gcount(), a, b);
      else 
        break;
    }
    file.close();
    
    const uint32 adler = (b << 16) | a;
    if (adler != fileRecord_->adler32)
    {
      std::ostringstream msg;
      msg << "Disk resident file " << fileName <<
        " has Adler32 checksum " << std::hex << adler <<
        ", while the expected checkum is " << std::hex << fileRecord_->adler32;
      XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
    }
  }

void stor::FileHandler::checkDirectories ( ) const [private]

Check if all directories needed for the file output are available. Throws a stor::execption::NoSuchDirectory when a directory does not exist.

Definition at line 333 of file FileHandler.cc.

References stor::utils::checkDirectory(), diskWritingParams_, stor::DiskWritingParams::filePath_, and fileRecord_.

  {
    utils::checkDirectory(diskWritingParams_.filePath_);
    utils::checkDirectory(fileRecord_->baseFilePath);
    utils::checkDirectory(fileRecord_->baseFilePath + "/open");
    utils::checkDirectory(fileRecord_->baseFilePath + "/closed");
  }

uint64_t stor::FileHandler::checkFileSizeMatch	(	const std::string &	fileName,
		const uint64_t &	size
	)		const `[private]`

Check that the file size matches the given size. Returns the actual size.

Definition at line 219 of file FileHandler.cc.

References fileRecord_, stor::FilesMonitorCollection::FileRecord::inaccessible, MatrixRunner::msg, sizeMismatch(), and stor::FilesMonitorCollection::FileRecord::truncated.

  {
    #if linux
    struct stat64 statBuff;
    int statStatus = stat64(fileName.c_str(), &statBuff);
    #else
    struct stat statBuff;
    int statStatus = stat(fileName.c_str(), &statBuff);
    #endif
    if ( statStatus != 0 )
    {
      fileRecord_->whyClosed = FilesMonitorCollection::FileRecord::inaccessible;
      std::ostringstream msg;
      msg << "Error checking the status of file "
        << fileName
        << ": " << strerror(errno);
      XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
    }
    
    if ( sizeMismatch(size, statBuff.st_size) )
    {
      fileRecord_->whyClosed = FilesMonitorCollection::FileRecord::truncated;
      std::ostringstream msg;
      msg << "Found an unexpected file size when trying to move"
        << " the file to the closed state. File " << fileName
        << " has an actual size of " << statBuff.st_size
        << " (" << statBuff.st_blocks << " blocks)"
        << " instead of the expected size of " << size
        << " (" << (size/512)+1 << " blocks).";
      XCEPT_RAISE(stor::exception::FileTruncation, msg.str());
    }
    
    return static_cast<uint64_t>(statBuff.st_size);
  }

virtual void stor::FileHandler::closeFile ( const FilesMonitorCollection::FileRecord::ClosingReason & ) [pure virtual]

Close the file

Implemented in stor::EventFileHandler, and stor::FRDFileHandler.

Referenced by isFromLumiSection(), tooLarge(), and tooOld().

virtual void stor::FileHandler::do_writeEvent ( const I2OChain & event ) [protected, pure virtual]

Write the I2OChain to the file

Implemented in stor::EventFileHandler, and stor::FRDFileHandler.

Referenced by writeEvent().

uint32_t stor::FileHandler::events ( ) const

Return the number of events in the file

Definition at line 175 of file FileHandler.cc.

References fileRecord_.

Referenced by insertFileInDatabase(), tooLarge(), and updateDatabase().

  {
    return fileRecord_->eventCount;
  }

uint64_t stor::FileHandler::fileSize ( ) const

Return the size of the file in bytes

Definition at line 181 of file FileHandler.cc.

References fileRecord_.

Referenced by insertFileInDatabase(), tooLarge(), and updateDatabase().

  {
    return fileRecord_->fileSize;
  }

void stor::FileHandler::insertFileInDatabase ( ) const [protected]

Write command to insert a new file into the CMS_STOMGR.TIER0_INJECTION table into the logFile_.

Definition at line 102 of file FileHandler.cc.

References cmsver_, dbFileHandler_, diskWritingParams_, events(), fileRecord_, fileSize(), firstEntry_, stor::DiskWritingParams::hostName_, stor::DiskWritingParams::initialSafetyLevel_, stor::utils::secondsSinceEpoch(), stor::DiskWritingParams::setupLabel_, and stor::DiskWritingParams::smInstanceString_.

  {
    std::ostringstream oss;
    oss << "./insertFile.pl "
      << " --FILENAME "     << fileRecord_->fileName()
      << " --FILECOUNTER "  << fileRecord_->fileCounter
      << " --NEVENTS "      << events()
      << " --FILESIZE "     << fileSize()
      << " --STARTTIME "    << utils::secondsSinceEpoch(firstEntry_)
      << " --STOPTIME 0"
      << " --STATUS open"
      << " --RUNNUMBER "    << fileRecord_->runNumber
      << " --LUMISECTION "  << fileRecord_->lumiSection
      << " --PATHNAME "     << fileRecord_->filePath()
      << " --HOSTNAME "     << diskWritingParams_.hostName_
      << " --SETUPLABEL "   << diskWritingParams_.setupLabel_
      << " --STREAM "       << fileRecord_->streamLabel
      << " --INSTANCE "     << diskWritingParams_.smInstanceString_
      << " --SAFETY "       << diskWritingParams_.initialSafetyLevel_
      << " --APPVERSION "   << cmsver_
      << " --APPNAME CMSSW"
      << " --TYPE streamer"               
      << " --CHECKSUM 0"
      << " --CHECKSUMIND 0"
      << "\n";
    
    dbFileHandler_->writeOld( firstEntry_, oss.str() );
  }

bool stor::FileHandler::isFromLumiSection ( const uint32_t lumiSection )

Returns true if the file corresponds to the given lumi section

Definition at line 147 of file FileHandler.cc.

References closeFile(), fileRecord_, and stor::FilesMonitorCollection::FileRecord::LSended.

Referenced by stor::StreamHandler::closeFilesForLumiSection().

  {
    if (lumiSection == fileRecord_->lumiSection)
    {
      closeFile(FilesMonitorCollection::FileRecord::LSended);
      return true;
    }
    else
    {
      return false;
    }
  }

std::string stor::FileHandler::logFile ( const DiskWritingParams & ) const [private]

Return the name of the log file

uint32_t stor::FileHandler::lumiSection ( ) const [inline]

Return the luminosity section the file belongs to

Definition at line 80 of file FileHandler.h.

References fileRecord_.

    { return fileRecord_->lumiSection; }

void stor::FileHandler::makeFileReadOnly ( const std::string & fileName ) const [private]

Changes the file permissions to read-only

Definition at line 262 of file FileHandler.cc.

References MatrixRunner::msg.

  {
    int ronly  = chmod(fileName.c_str(), S_IREAD|S_IRGRP|S_IROTH);
    if (ronly != 0) {
      std::ostringstream msg;
      msg << "Unable to change permissions of " << fileName
        << " to read only: " << strerror(errno);
      XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
    }
  }

void stor::FileHandler::moveFileToClosed ( const FilesMonitorCollection::FileRecord::ClosingReason & reason ) [protected]

Move index and streamer file to "closed" directory

Definition at line 193 of file FileHandler.cc.

References stor::FilesMonitorCollection::FileRecord::closed, stor::FilesMonitorCollection::FileRecord::open, and Association::reason.

Referenced by stor::EventFileHandler::closeFile(), and stor::FRDFileHandler::closeFile().

  {
    const std::string openFileName(fileRecord_->completeFileName(FilesMonitorCollection::FileRecord::open));
    const std::string closedFileName(fileRecord_->completeFileName(FilesMonitorCollection::FileRecord::closed));
    
    const uint64_t openFileSize = checkFileSizeMatch(openFileName, fileSize());
    
    makeFileReadOnly(openFileName);
    try
    {
      renameFile(openFileName, closedFileName);
    }
    catch (stor::exception::DiskWriting& e)
    {
      XCEPT_RETHROW(stor::exception::DiskWriting, 
        "Could not move streamer file to closed area.", e);
    }
    fileRecord_->isOpen = false;
    fileRecord_->whyClosed = reason;
    checkFileSizeMatch(closedFileName, openFileSize);
    checkAdler32(closedFileName);
  }

FileHandler& stor::FileHandler::operator= ( FileHandler const & ) [private]

void stor::FileHandler::renameFile	(	const std::string &	openFileName,
		const std::string &	closedFileName
	)		const `[private]`

Rename the file

Definition at line 317 of file FileHandler.cc.

References MatrixRunner::msg, stor::FilesMonitorCollection::FileRecord::notClosed, and query::result.

  {
    int result = rename( openFileName.c_str(), closedFileName.c_str() );
    if (result != 0) {
      fileRecord_->whyClosed = FilesMonitorCollection::FileRecord::notClosed;
      std::ostringstream msg;
      msg << "Unable to move " << openFileName << " to "
        << closedFileName << ": " << strerror(errno);
      XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
    }
  }

bool stor::FileHandler::sizeMismatch	(	const uint64_t &	initialSize,
		const uint64_t &	finalSize
	)		const `[private]`

Check that the 2 sizes agree

Definition at line 255 of file FileHandler.cc.

References calcPctDiff(), diskWritingParams_, and stor::DiskWritingParams::fileSizeTolerance_.

Referenced by checkFileSizeMatch().

  {
    double pctDiff = calcPctDiff(initialSize, finalSize);
    return (pctDiff > diskWritingParams_.fileSizeTolerance_);
  }

bool stor::FileHandler::tooLarge ( const uint64_t & dataSize )

Returns true if the additional data size would push the file size beyond maxFileSize.

Definition at line 161 of file FileHandler.cc.

References closeFile(), events(), fileSize(), maxFileSize_, and stor::FilesMonitorCollection::FileRecord::size.

  {
    if ( ((fileSize() + dataSize) > maxFileSize_) && (events() > 0) )
    {
      closeFile(FilesMonitorCollection::FileRecord::size);
      return true;
    }
    else
    {
      return false;
    }
  }

bool stor::FileHandler::tooOld ( const utils::TimePoint_t currentTime = utils::getCurrentTime() )

Returns true if the file has not seen any recent events

Definition at line 132 of file FileHandler.cc.

References closeFile(), diskWritingParams_, lastEntry_, stor::DiskWritingParams::lumiSectionTimeOut_, seconds(), and stor::FilesMonitorCollection::FileRecord::timeout.

Referenced by stor::StreamHandler::closeTimedOutFiles().

  {
    if (diskWritingParams_.lumiSectionTimeOut_ > boost::posix_time::seconds(0) && 
      (currentTime - lastEntry_) > diskWritingParams_.lumiSectionTimeOut_)
    {
      closeFile(FilesMonitorCollection::FileRecord::timeout);
      return true;
    }
    else
    {
      return false;
    }
  }

void stor::FileHandler::updateDatabase ( ) const [protected]

Write command to update the file information in the CMS_STOMGR.TIER0_INJECTION table into the logFile_.

Definition at line 71 of file FileHandler.cc.

References cmsver_, dbFileHandler_, diskWritingParams_, events(), fileRecord_, fileSize(), firstEntry_, stor::DiskWritingParams::hostName_, stor::DiskWritingParams::initialSafetyLevel_, lastEntry_, stor::utils::secondsSinceEpoch(), stor::DiskWritingParams::setupLabel_, and stor::DiskWritingParams::smInstanceString_.

Referenced by stor::EventFileHandler::closeFile(), and stor::FRDFileHandler::closeFile().

  {
    std::ostringstream oss;
    oss << "./closeFile.pl "
      << " --FILENAME "     << fileRecord_->fileName()
      << " --FILECOUNTER "  << fileRecord_->fileCounter
      << " --NEVENTS "      << events()
      << " --FILESIZE "     << fileSize()                          
      << " --STARTTIME "    << utils::secondsSinceEpoch(firstEntry_)
      << " --STOPTIME "     << utils::secondsSinceEpoch(lastEntry_)
      << " --STATUS "       << "closed"
      << " --RUNNUMBER "    << fileRecord_->runNumber
      << " --LUMISECTION "  << fileRecord_->lumiSection
      << " --PATHNAME "     << fileRecord_->filePath()
      << " --HOSTNAME "     << diskWritingParams_.hostName_
      << " --SETUPLABEL "   << diskWritingParams_.setupLabel_
      << " --STREAM "       << fileRecord_->streamLabel                      
      << " --INSTANCE "     << diskWritingParams_.smInstanceString_
      << " --SAFETY "       << diskWritingParams_.initialSafetyLevel_
      << " --APPVERSION "   << cmsver_
      << " --APPNAME CMSSW"
      << " --TYPE streamer"               
      << " --DEBUGCLOSE "   << fileRecord_->whyClosed
      << " --CHECKSUM "     << std::hex << fileRecord_->adler32
      << " --CHECKSUMIND 0"
      << "\n";
    
    dbFileHandler_->writeOld( lastEntry_, oss.str() );
  }

void stor::FileHandler::writeEvent ( const I2OChain & event )

Write the event contained in the I2OChain

Definition at line 48 of file FileHandler.cc.

References do_writeEvent(), fileRecord_, stor::utils::getCurrentTime(), lastEntry_, and MatrixRunner::msg.

  {
    if ( ! fileRecord_->isOpen )
    {
      std::ostringstream msg;
    msg << "Tried to write an event to "
      << fileRecord_->completeFileName()
      << "which has already been closed.";
    XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
    }
    
    do_writeEvent(event);
    
    fileRecord_->fileSize += event.totalDataSize();
    ++fileRecord_->eventCount;
    lastEntry_ = utils::getCurrentTime();
  }