#include <FileHandler.h>

Inheritance diagram for stor::FileHandler:

Public Member Functions
virtual void	closeFile (const FilesMonitorCollection::FileRecord::ClosingReason &)=0

uint32_t	events () const

	FileHandler (FilesMonitorCollection::FileRecordPtr, const DbFileHandlerPtr, const DiskWritingParams &, const uint64_t &maxFileSize)

uint64_t	fileSize () const

bool	isFromLumiSection (const uint32_t lumiSection)

uint32_t	lumiSection () const

bool	tooLarge (const uint64_t &dataSize)

bool	tooOld (const utils::TimePoint_t currentTime=utils::getCurrentTime())

void	writeEvent (const I2OChain &)

virtual	~FileHandler ()

Protected Member Functions
virtual void	do_writeEvent (const I2OChain &event)=0

void	insertFileInDatabase () const

void	moveFileToClosed (const FilesMonitorCollection::FileRecord::ClosingReason &)

void	updateDatabase () const

Protected Attributes
const DbFileHandlerPtr	dbFileHandler_

const DiskWritingParams &	diskWritingParams_

FilesMonitorCollection::FileRecordPtr	fileRecord_

utils::TimePoint_t	firstEntry_

utils::TimePoint_t	lastEntry_

Private Member Functions
double	calcPctDiff (const uint64_t &, const uint64_t &) const

void	checkAdler32 (const std::string &fileName) const

void	checkDirectories () const

uint64_t	checkFileSizeMatch (const std::string &fileName, const uint64_t &size) const

	FileHandler (FileHandler const &)

std::string	logFile (const DiskWritingParams &) const

void	makeFileReadOnly (const std::string &fileName) const

FileHandler &	operator= (FileHandler const &)

void	renameFile (const std::string &openFileName, const std::string &closedFileName) const

bool	sizeMismatch (const uint64_t &initialSize, const uint64_t &finalSize) const

Private Attributes
std::string	cmsver_

const std::string	logFile_

const std::string	logPath_

const uint64_t	maxFileSize_

Detailed Description

Abstract representation of a physical file

Author:: mommsen

Revision:: 1.19

Date:: 2011/07/05 13:25:43

Definition at line 32 of file FileHandler.h.

Constructor & Destructor Documentation

stor::FileHandler::FileHandler	(	FilesMonitorCollection::FileRecordPtr	fileRecord,
		const DbFileHandlerPtr	dbFileHandler,
		const DiskWritingParams &	dwParams,
		const uint64_t &	maxFileSize
	)

Definition at line 26 of file FileHandler.cc.

    :
   fileRecord_(fileRecord),
   dbFileHandler_(dbFileHandler),
   firstEntry_(utils::getCurrentTime()),
   lastEntry_(firstEntry_),
   diskWritingParams_(dwParams),
   maxFileSize_(maxFileSize),
   cmsver_(edm::getReleaseVersion())
   {
     // stripp quotes if present
     if(cmsver_[0]=='"') cmsver_=cmsver_.substr(1,cmsver_.size()-2);
     
     checkDirectories();
     
     insertFileInDatabase();
   }

virtual stor::FileHandler::~FileHandler ( )

inlinevirtual

Definition at line 44 of file FileHandler.h.

44 {};

stor::FileHandler::FileHandler ( FileHandler const & )

private

Member Function Documentation

double stor::FileHandler::calcPctDiff	(	const uint64_t &	value1,
		const uint64_t &	value2
	)		const

private

Return the relative difference btw to file sizes

Definition at line 342 of file FileHandler.cc.

References max(), and min.

Referenced by sizeMismatch().

   {
     if (value1 == value2) return 0;
     uint64_t largerValue = std::max(value1,value2);
     uint64_t smallerValue = std::min(value1,value2);
     return ( largerValue > 0 ? (largerValue - smallerValue) / largerValue : 0 );
   }

void stor::FileHandler::checkAdler32 ( const std::string & fileName ) const

private

Compare Adler32 checksum of file with calculated one

Definition at line 274 of file FileHandler.cc.

References a, cms::Adler32(), b, stor::DiskWritingParams::checkAdler32_, diskWritingParams_, mergeVDriftHistosByStation::file, fileRecord_, recoMuon::in, runTheMatrix::msg, and findQualityFiles::size.

   {
     if (!diskWritingParams_.checkAdler32_) return;
 
     std::ifstream file(fileName.c_str(), std::ios::in | std::ios::binary | std::ios::ate);
     if (!file.is_open())
     {
       std::ostringstream msg;
       msg << "File " << fileName << " could not be opened.\n";
       XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
     }
     
     std::ifstream::pos_type size = file.tellg();
     file.seekg (0, std::ios::beg);
     
     boost::shared_array<char> ptr(new char[1024*1024]);
     uint32_t a = 1, b = 0;
     
     std::ifstream::pos_type rsize = 0;
     while(rsize < size)
     {
       file.read(ptr.get(), 1024*1024);
       rsize += 1024*1024;
       if(file.gcount()) 
         cms::Adler32(ptr.get(), file.gcount(), a, b);
       else 
         break;
     }
     file.close();
     
     const uint32 adler = (b << 16) | a;
     if (adler != fileRecord_->adler32)
     {
       std::ostringstream msg;
       msg << "Disk resident file " << fileName <<
         " has Adler32 checksum " << std::hex << adler <<
         ", while the expected checkum is " << std::hex << fileRecord_->adler32;
       XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
     }
   }

void stor::FileHandler::checkDirectories ( ) const

private

Check if all directories needed for the file output are available. Throws a stor::execption::NoSuchDirectory when a directory does not exist.

Definition at line 333 of file FileHandler.cc.

References stor::utils::checkDirectory(), diskWritingParams_, stor::DiskWritingParams::filePath_, and fileRecord_.

   {
     utils::checkDirectory(diskWritingParams_.filePath_);
     utils::checkDirectory(fileRecord_->baseFilePath);
     utils::checkDirectory(fileRecord_->baseFilePath + "/open");
     utils::checkDirectory(fileRecord_->baseFilePath + "/closed");
   }

uint64_t stor::FileHandler::checkFileSizeMatch	(	const std::string &	fileName,
		const uint64_t &	size
	)		const

private

Check that the file size matches the given size. Returns the actual size.

Definition at line 219 of file FileHandler.cc.

References fileRecord_, stor::FilesMonitorCollection::FileRecord::inaccessible, runTheMatrix::msg, sizeMismatch(), and stor::FilesMonitorCollection::FileRecord::truncated.

   {
     #if linux
     struct stat64 statBuff;
     int statStatus = stat64(fileName.c_str(), &statBuff);
     #else
     struct stat statBuff;
     int statStatus = stat(fileName.c_str(), &statBuff);
     #endif
     if ( statStatus != 0 )
     {
       fileRecord_->whyClosed = FilesMonitorCollection::FileRecord::inaccessible;
       std::ostringstream msg;
       msg << "Error checking the status of file "
         << fileName
         << ": " << strerror(errno);
       XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
     }
     
     if ( sizeMismatch(size, statBuff.st_size) )
     {
       fileRecord_->whyClosed = FilesMonitorCollection::FileRecord::truncated;
       std::ostringstream msg;
       msg << "Found an unexpected file size when trying to move"
         << " the file to the closed state. File " << fileName
         << " has an actual size of " << statBuff.st_size
         << " (" << statBuff.st_blocks << " blocks)"
         << " instead of the expected size of " << size
         << " (" << (size/512)+1 << " blocks).";
       XCEPT_RAISE(stor::exception::FileTruncation, msg.str());
     }
     
     return static_cast<uint64_t>(statBuff.st_size);
   }

virtual void stor::FileHandler::closeFile ( const FilesMonitorCollection::FileRecord::ClosingReason & )

pure virtual

Close the file

Implemented in stor::EventFileHandler, and stor::FRDFileHandler.

Referenced by isFromLumiSection(), python.Vispa.Main.Application.Application::tabCloseRequest(), tooLarge(), and tooOld().

virtual void stor::FileHandler::do_writeEvent ( const I2OChain & event )

protectedpure virtual

Write the I2OChain to the file

Implemented in stor::EventFileHandler, and stor::FRDFileHandler.

Referenced by writeEvent().

uint32_t stor::FileHandler::events ( ) const

Return the number of events in the file

Definition at line 175 of file FileHandler.cc.

References fileRecord_.

Referenced by insertFileInDatabase(), tooLarge(), and updateDatabase().

   {
     return fileRecord_->eventCount;
   }

uint64_t stor::FileHandler::fileSize ( ) const

Return the size of the file in bytes

Definition at line 181 of file FileHandler.cc.

References fileRecord_.

Referenced by insertFileInDatabase(), tooLarge(), and updateDatabase().

   {
     return fileRecord_->fileSize;
   }

void stor::FileHandler::insertFileInDatabase ( ) const

protected

Write command to insert a new file into the CMS_STOMGR.TIER0_INJECTION table into the logFile_.

Definition at line 102 of file FileHandler.cc.

References cmsver_, dbFileHandler_, diskWritingParams_, events(), fileRecord_, fileSize(), firstEntry_, stor::DiskWritingParams::hostName_, stor::DiskWritingParams::initialSafetyLevel_, stor::utils::secondsSinceEpoch(), stor::DiskWritingParams::setupLabel_, and stor::DiskWritingParams::smInstanceString_.

   {
     std::ostringstream oss;
     oss << "./insertFile.pl "
       << " --FILENAME "     << fileRecord_->fileName()
       << " --FILECOUNTER "  << fileRecord_->fileCounter
       << " --NEVENTS "      << events()
       << " --FILESIZE "     << fileSize()
       << " --STARTTIME "    << utils::secondsSinceEpoch(firstEntry_)
       << " --STOPTIME 0"
       << " --STATUS open"
       << " --RUNNUMBER "    << fileRecord_->runNumber
       << " --LUMISECTION "  << fileRecord_->lumiSection
       << " --PATHNAME "     << fileRecord_->filePath()
       << " --HOSTNAME "     << diskWritingParams_.hostName_
       << " --SETUPLABEL "   << diskWritingParams_.setupLabel_
       << " --STREAM "       << fileRecord_->streamLabel
       << " --INSTANCE "     << diskWritingParams_.smInstanceString_
       << " --SAFETY "       << diskWritingParams_.initialSafetyLevel_
       << " --APPVERSION "   << cmsver_
       << " --APPNAME CMSSW"
       << " --TYPE streamer"               
       << " --CHECKSUM 0"
       << " --CHECKSUMIND 0"
       << "\n";
     
     dbFileHandler_->writeOld( firstEntry_, oss.str() );
   }

bool stor::FileHandler::isFromLumiSection ( const uint32_t lumiSection )

Returns true if the file corresponds to the given lumi section

Definition at line 147 of file FileHandler.cc.

References closeFile(), fileRecord_, and stor::FilesMonitorCollection::FileRecord::LSended.

Referenced by stor::StreamHandler::closeFilesForLumiSection().

   {
     if (lumiSection == fileRecord_->lumiSection)
     {
       closeFile(FilesMonitorCollection::FileRecord::LSended);
       return true;
     }
     else
     {
       return false;
     }
   }

std::string stor::FileHandler::logFile ( const DiskWritingParams & ) const

private

Return the name of the log file

uint32_t stor::FileHandler::lumiSection ( ) const

inline

Return the luminosity section the file belongs to

Definition at line 80 of file FileHandler.h.

References fileRecord_.

81 { return fileRecord_->lumiSection; }

stor::FileHandler::fileRecord_

FilesMonitorCollection::FileRecordPtr fileRecord_

Definition: FileHandler.h:187

void stor::FileHandler::makeFileReadOnly ( const std::string & fileName ) const

private

Changes the file permissions to read-only

Definition at line 262 of file FileHandler.cc.

References runTheMatrix::msg.

   {
     int ronly  = chmod(fileName.c_str(), S_IREAD|S_IRGRP|S_IROTH);
     if (ronly != 0) {
       std::ostringstream msg;
       msg << "Unable to change permissions of " << fileName
         << " to read only: " << strerror(errno);
       XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
     }
   }

void stor::FileHandler::moveFileToClosed ( const FilesMonitorCollection::FileRecord::ClosingReason & reason )

protected

Move index and streamer file to "closed" directory

Definition at line 193 of file FileHandler.cc.

References stor::FilesMonitorCollection::FileRecord::closed, and stor::FilesMonitorCollection::FileRecord::open.

Referenced by stor::FRDFileHandler::closeFile(), and stor::EventFileHandler::closeFile().

   {
     const std::string openFileName(fileRecord_->completeFileName(FilesMonitorCollection::FileRecord::open));
     const std::string closedFileName(fileRecord_->completeFileName(FilesMonitorCollection::FileRecord::closed));
     
     const uint64_t openFileSize = checkFileSizeMatch(openFileName, fileSize());
     
     makeFileReadOnly(openFileName);
     try
     {
       renameFile(openFileName, closedFileName);
     }
     catch (stor::exception::DiskWriting& e)
     {
       XCEPT_RETHROW(stor::exception::DiskWriting, 
         "Could not move streamer file to closed area.", e);
     }
     fileRecord_->isOpen = false;
     fileRecord_->whyClosed = reason;
     checkFileSizeMatch(closedFileName, openFileSize);
     checkAdler32(closedFileName);
   }

FileHandler& stor::FileHandler::operator= ( FileHandler const & )

private

void stor::FileHandler::renameFile	(	const std::string &	openFileName,
		const std::string &	closedFileName
	)		const

private

Rename the file

Definition at line 317 of file FileHandler.cc.

References runTheMatrix::msg, stor::FilesMonitorCollection::FileRecord::notClosed, and query::result.

   {
     int result = rename( openFileName.c_str(), closedFileName.c_str() );
     if (result != 0) {
       fileRecord_->whyClosed = FilesMonitorCollection::FileRecord::notClosed;
       std::ostringstream msg;
       msg << "Unable to move " << openFileName << " to "
         << closedFileName << ": " << strerror(errno);
       XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
     }
   }

bool stor::FileHandler::sizeMismatch	(	const uint64_t &	initialSize,
		const uint64_t &	finalSize
	)		const

private

Check that the 2 sizes agree

Definition at line 255 of file FileHandler.cc.

References calcPctDiff(), diskWritingParams_, and stor::DiskWritingParams::fileSizeTolerance_.

Referenced by checkFileSizeMatch().

   {
     double pctDiff = calcPctDiff(initialSize, finalSize);
     return (pctDiff > diskWritingParams_.fileSizeTolerance_);
   }

bool stor::FileHandler::tooLarge ( const uint64_t & dataSize )

Returns true if the additional data size would push the file size beyond maxFileSize.

Definition at line 161 of file FileHandler.cc.

References closeFile(), events(), fileSize(), maxFileSize_, and stor::FilesMonitorCollection::FileRecord::size.

   {
     if ( ((fileSize() + dataSize) > maxFileSize_) && (events() > 0) )
     {
       closeFile(FilesMonitorCollection::FileRecord::size);
       return true;
     }
     else
     {
       return false;
     }
   }

bool stor::FileHandler::tooOld ( const utils::TimePoint_t currentTime = utils::getCurrentTime() )

Returns true if the file has not seen any recent events

Definition at line 132 of file FileHandler.cc.

References closeFile(), diskWritingParams_, lastEntry_, stor::DiskWritingParams::lumiSectionTimeOut_, seconds(), and stor::FilesMonitorCollection::FileRecord::timeout.

Referenced by stor::StreamHandler::closeTimedOutFiles().

   {
     if (diskWritingParams_.lumiSectionTimeOut_ > boost::posix_time::seconds(0) && 
       (currentTime - lastEntry_) > diskWritingParams_.lumiSectionTimeOut_)
     {
       closeFile(FilesMonitorCollection::FileRecord::timeout);
       return true;
     }
     else
     {
       return false;
     }
   }

void stor::FileHandler::updateDatabase ( ) const

protected

Write command to update the file information in the CMS_STOMGR.TIER0_INJECTION table into the logFile_.

Definition at line 71 of file FileHandler.cc.

References cmsver_, dbFileHandler_, diskWritingParams_, events(), fileRecord_, fileSize(), firstEntry_, stor::DiskWritingParams::hostName_, stor::DiskWritingParams::initialSafetyLevel_, lastEntry_, stor::utils::secondsSinceEpoch(), stor::DiskWritingParams::setupLabel_, and stor::DiskWritingParams::smInstanceString_.

Referenced by stor::FRDFileHandler::closeFile(), and stor::EventFileHandler::closeFile().

   {
     std::ostringstream oss;
     oss << "./closeFile.pl "
       << " --FILENAME "     << fileRecord_->fileName()
       << " --FILECOUNTER "  << fileRecord_->fileCounter
       << " --NEVENTS "      << events()
       << " --FILESIZE "     << fileSize()                          
       << " --STARTTIME "    << utils::secondsSinceEpoch(firstEntry_)
       << " --STOPTIME "     << utils::secondsSinceEpoch(lastEntry_)
       << " --STATUS "       << "closed"
       << " --RUNNUMBER "    << fileRecord_->runNumber
       << " --LUMISECTION "  << fileRecord_->lumiSection
       << " --PATHNAME "     << fileRecord_->filePath()
       << " --HOSTNAME "     << diskWritingParams_.hostName_
       << " --SETUPLABEL "   << diskWritingParams_.setupLabel_
       << " --STREAM "       << fileRecord_->streamLabel                      
       << " --INSTANCE "     << diskWritingParams_.smInstanceString_
       << " --SAFETY "       << diskWritingParams_.initialSafetyLevel_
       << " --APPVERSION "   << cmsver_
       << " --APPNAME CMSSW"
       << " --TYPE streamer"               
       << " --DEBUGCLOSE "   << fileRecord_->whyClosed
       << " --CHECKSUM "     << std::hex << fileRecord_->adler32
       << " --CHECKSUMIND 0"
       << "\n";
     
     dbFileHandler_->writeOld( lastEntry_, oss.str() );
   }

void stor::FileHandler::writeEvent ( const I2OChain & event )

Write the event contained in the I2OChain

Definition at line 48 of file FileHandler.cc.

References do_writeEvent(), fileRecord_, stor::utils::getCurrentTime(), lastEntry_, and runTheMatrix::msg.

   {
     if ( ! fileRecord_->isOpen )
     {
       std::ostringstream msg;
     msg << "Tried to write an event to "
       << fileRecord_->completeFileName()
       << "which has already been closed.";
     XCEPT_RAISE(stor::exception::DiskWriting, msg.str());
     }
     
     do_writeEvent(event);
     
     fileRecord_->fileSize += event.totalDataSize();
     ++fileRecord_->eventCount;
     lastEntry_ = utils::getCurrentTime();
   }