Functions
def	_xrdcpSingleFile (pfn_src, pfn_dest)

def	cat (path)

def	chmod (path, mode)

def	cmsStage (absDestDir, files, force)

def	createEOSDir (path)

def	datasetNotEmpty (path, regexp)

def	eosDirSize (path)

def	eosToLFN (path)

def	fileExists (path)

def	isDirectory (path)

def	isEOS (path)

def	isEOSDir (path)

def	isEOSFile (path, tfcProt='rfio')

def	isFile (path)

def	isLFN (path)

def	lfnToEOS (path)

def	lfnToPFN (path, tfcProt='rfio')

def	listFiles (path, rec=False, full_info=False)

def	ls (path, rec=False)

def	ls_EOS (path, rec=False)

def	matchingFiles (path, regexp)

def	mkdir (path)

def	move (src, dest)

def	remove (files, rec=False)

def	rm (path, rec=False)

def	runEOSCommand (path, cmd, args)

def	runXRDCommand (path, cmd, args)

def	setCAFPath ()

def	which (cmd)

def	xrdcp (src, dest)

Variables
	castorToLFN

	createCastorDir

	eos_select

	isCastorDir

	isCastorFile

	lfnToCastor

Detailed Description

A module to manipulate files on EOS or on the local file system. Intended to have the same interface as castortools.py.

Function Documentation

def eostools._xrdcpSingleFile	(	pfn_src,
		pfn_dest
	)

private

Copies a single file using xrd.

Definition at line 495 of file eostools.py.

 def _xrdcpSingleFile( pfn_src, pfn_dest):
     """Copies a single file using xrd."""
     
     command = ['xrdcp']
     command.append(pfn_src)
     command.append(pfn_dest)
     # print ' '.join(command)
     run = True
     if run: 
         runner = cmsIO.cmsFileManip()
         out, err, ret = runner.runCommand(command)
         if err:
             print(out, file=sys.stderr)
             print(err, file=sys.stderr)
         return ret
 

def eostools.cat ( path )

cat, works on EOS and locally

Definition at line 401 of file eostools.py.

Referenced by EnergyScaleCorrection.addScale(), EnergyScaleCorrection_class.AddScale(), EnergyScaleCorrection.addSmearing(), EnergyScaleCorrection_class.AddSmearing(), Rivet::HiggsTemplateCrossSections.analyze(), MVAJetTagPlotter.analyzeTag(), dqm::TrackAnalyzer.bookHistosForEfficiencyFromHitPatter(), CSCStripAmpResponse.calculateAmpResponse(), HGCalTriggerClusterIdentificationBDT.category(), CutBasedElectronID.cicSelection(), CutBasedElectronID.classify(), Rivet::HiggsTemplateCrossSections.classifyEvent(), DDLogicalPart.DDLogicalPart(), HGCalTriggerClusterIdentificationBDT.decision(), Rivet::HiggsTemplateCrossSections.error(), fwlite::Scanner< Collection >.fillDataSet(), edm::service::ELstatistics.formSummary(), HGCalTriggerClusterIdentificationBDT.initialize(), PhoMVACut.operator()(), GsfEleMVACut.operator()(), GctFormatTranslateBase.packingEventId(), DDLLogicalPart.processElement(), MVAValueMapProducer< ParticleType >.produce(), HTXSRivetProducer.produce(), AlgorithmCalibration< T, CO >.readCategories(), HTXS.Rivet2Root(), TagProbeFitter.setWeightVar(), edm::service::ELstatistics.summaryForJobReport(), TagProbeFitTreeAnalyzer.TagProbeFitTreeAnalyzer(), and HGCalTriggerClusterIdentificationBDT.value().

 def cat(path):
     """cat, works on EOS and locally"""
     path = lfnToEOS(path)
     if isEOS(path):
         #print "the file to cat is:", path
         out, err, _ = runXRDCommand(path,'cat') 
         lines = []
         if out:
             pattern = re.compile('cat returned [0-9]+')
             for line in out.split('\n'):
                 match = pattern.search(line)
                 if line and match is not None:
                     lines.append(line.replace(match.group(0),''))
                     break
                 else:
                     lines.append(line)
         if err:
             print(out, file=sys.stderr)
             print(err, file=sys.stderr)
         allLines = '\n'.join(lines)
         if allLines and not allLines.endswith('\n'):
             allLines += '\n'
         return allLines
     else:
         content = file(path).read()
         if content and not content.endswith('\n'):
             content += '\n'
         return content
     

def eostools.chmod	(	path,
		mode
	)

Does chmod on a file or directory

Definition at line 294 of file eostools.py.

Referenced by evf::EvFDaqDirector.initRun(), L1GtVhdlWriterCore.makeFirmware(), evf::EvFDaqDirector.openFULockfileStream(), L1GtVhdlWriterCore.writeAlgoSetup(), L1GtVhdlWriterCore.writeCond2intMap2File(), L1GtVhdlWriterCore.writeCondChipPkg(), L1GtVhdlWriterCore.writeConditionChipSetup(), L1GtVhdlWriterCore.writeDefValPkg(), L1GtVhdlWriterCore.writeEtmSetup(), L1GtVhdlWriterCore.writeMuonSetupVhdl(), and L1GtVhdlWriterCore.writeQsfSetupFiles().

 def chmod(path, mode):
     """Does chmod on a file or directory"""
     #
     return runEOSCommand(path, 'chmod', '-r', str(mode))
 
 

def eostools.cmsStage	(	absDestDir,
		files,
		force
	)

Runs cmsStage with LFNs if possible

Definition at line 538 of file eostools.py.

 def cmsStage( absDestDir, files, force):
     """Runs cmsStage with LFNs if possible"""
 
     destIsEOSDir = isEOSDir(absDestDir)
     if destIsEOSDir: 
         createEOSDir( absDestDir )
 
     for fname in files:
         command = ['cmsStage']
         if force:
             command.append('-f')
         command.append(eosToLFN(fname))
         command.append(eosToLFN(absDestDir))
         print(' '.join(command))
         runner = cmsIO.cmsFileManip()
         runner.runCommand(command)

def eostools.createEOSDir ( path )

Makes a directory in EOS

???Will, I'm quite worried by the fact that if this path already exists, and is
a file, everything will 'work'. But then we have a file, and not a directory,
while we expect a dir...

Definition at line 230 of file eostools.py.

Referenced by cmsStage().

 def createEOSDir( path ):
     """Makes a directory in EOS
 
     ???Will, I'm quite worried by the fact that if this path already exists, and is
     a file, everything will 'work'. But then we have a file, and not a directory,
     while we expect a dir..."""
     lfn = eosToLFN(path)
     if not isEOSFile(lfn):
     # if not isDirectory(lfn):
         runEOSCommand(lfn,'mkdir','-p')
         #        entity = cmsIO.cmsFile( lfn,"stageout")
         #        entity.mkdir([])
         #        # print 'created ', path
     if isDirectory(path):
         return path
     else:
         raise OSError('cannot create directory '+ path)
 
 #also define an alias for backwards compatibility

def eostools.datasetNotEmpty	(	path,
		regexp
	)

Definition at line 529 of file eostools.py.

 def datasetNotEmpty( path, regexp ):
     pattern = re.compile( regexp )
     files = ls_EOS(path)
     
     for f in files:
         if pattern.match( os.path.basename(f) ) is not None:
             return 1
     return 0
     

def eostools.eosDirSize ( path )

Returns the size of a directory on EOS in GB.

Definition at line 216 of file eostools.py.

 def eosDirSize(path):
     '''Returns the size of a directory on EOS in GB.'''
     lfn = eosToLFN(path)
     res = runEOSCommand(lfn, 'find', '--size')
     output = res[0].split('\n')
     size = 0
     for file in output:
         try:
             size += float(file.split('=')[2])
         except IndexError:
             pass
     return size/1024/1024/1024
 
 

def eostools.eosToLFN ( path )

Converts a EOS PFN to an LFN.

Just strip out /eos/cms from path.
If this string is not found, return path.
??? Shouldn't we raise an exception instead?

Definition at line 66 of file eostools.py.

References python.rootplot.root2matplotlib.replace().

Referenced by cmsStage(), createEOSDir(), eosDirSize(), isEOSDir(), cmsBatch.rootfiles_to_eos_script(), runEOSCommand(), and runXRDCommand().

 def eosToLFN( path ):
     """Converts a EOS PFN to an LFN.
 
     Just strip out /eos/cms from path.
     If this string is not found, return path.
     ??? Shouldn't we raise an exception instead?"""
     return path.replace('root://eoscms.cern.ch/', '').replace('/eos/cms','')
 
 #also define an alias for backwards compatibility

def eostools.fileExists ( path )

Returns true if path is a file or directory stored locally, or on EOS.

This function checks for the file or directory existence.

Definition at line 198 of file eostools.py.

 def fileExists( path ):
     """Returns true if path is a file or directory stored locally, or on EOS.
 
     This function checks for the file or directory existence."""
 
     eos = isEOSDir(path)
     result = False
     if eos:
         # print 'eos', path
         result = isEOSFile(path)
     else:
         # print 'not eos', path
         #check locally
         result = os.path.exists(path)
     # print result
     return result
 
 

def eostools.isDirectory ( path )

Returns True if path is a directory on EOS.

Tests for file existence. 
This function returns False for EOS files, and crashes with local paths

???Will, this function also seems to work for paths like:
/eos/cms/...
??? I think that it should work also for local files, see isFile.

Definition at line 261 of file eostools.py.

Referenced by createEOSDir().

 def isDirectory(path):
     """Returns True if path is a directory on EOS.
 
     Tests for file existence. 
     This function returns False for EOS files, and crashes with local paths
 
     ???Will, this function also seems to work for paths like:
     /eos/cms/...
     ??? I think that it should work also for local files, see isFile."""
 
     out, _, _ = runXRDCommand(path,'existdir')
     return 'The directory exists' in out
 

def eostools.isEOS ( path )

Tests whether this path is a CMS EOS (name starts with /eos...)

Definition at line 62 of file eostools.py.

Referenced by cat().

 def isEOS( path ):
     """Tests whether this path is a CMS EOS (name starts with /eos...)"""
     return path.startswith('/eos') or path.startswith('root://eoscms.cern.ch//eos/cms')
 

def eostools.isEOSDir ( path )

Returns True if path is either:
/store/...
or
/eos/cms/store/...
or
root://eoscms.cern.ch//eos/cms/

Otherwise, returns False.

WARNING!! This function does not check for path existence,
and returns true also for plain files.
!!! Will, is my summary correct?

Definition at line 122 of file eostools.py.

References eosToLFN(), and lfnToPFN().

Referenced by cmsStage(), and fileExists().

 def isEOSDir( path ):
     """Returns True if path is either:
     /store/...
     or
     /eos/cms/store/...
     or
     root://eoscms.cern.ch//eos/cms/
 
     Otherwise, returns False.
 
     WARNING!! This function does not check for path existence,
     and returns true also for plain files.
     !!! Will, is my summary correct? 
     """
     if os.path.exists( path ):
         # path does not exist
         # COLIN: I think this condition could be removed,
         # as it duplicates the following one. 
         return False
     if not path.startswith('/eos') and not path.startswith('/store') and not path.startswith('root://eoscms.cern.ch//eos/cms/'):
         # neither an EOS PFN or a LFN.
         return False
     # at this stage, we must have an EOS PFN or an LFN
     pfn = lfnToPFN(eosToLFN(path))
     tokens = cmsIO.splitPFN(pfn)
     return tokens and tokens[1].lower().startswith('eos')
 
 #also define an alias for backwards compatibility

def eostools.isEOSFile	(	path,
		tfcProt = `'rfio'`
	)

Returns True if path is a file or directory stored on EOS (checks for path existence)
??? This function does not behave well if passed a non EOS path...
returns lots of error messages like:
>>> eostools.isEOSFile('/store/asdfasfd')
Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2
ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory

ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory

False

Definition at line 153 of file eostools.py.

Referenced by createEOSDir(), and fileExists().

 def isEOSFile( path, tfcProt = 'rfio'):
     """Returns True if path is a file or directory stored on EOS (checks for path existence)
     ??? This function does not behave well if passed a non EOS path...
     returns lots of error messages like:
 >>> eostools.isEOSFile('/store/asdfasfd')
 Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2
 ls: s: No such file or directory
 ls: t: No such file or directory
 ls: o: No such file or directory
 ls: r: No such file or directory
 ls: e: No such file or directory
 ls: a: No such file or directory
 ls: s: No such file or directory
 ls: d: No such file or directory
 ls: f: No such file or directory
 ls: a: No such file or directory
 ls: s: No such file or directory
 ls: f: No such file or directory
 ls: d: No such file or directory
 ls: /store: No such file or directory
 
 ls: s: No such file or directory
 ls: t: No such file or directory
 ls: o: No such file or directory
 ls: r: No such file or directory
 ls: e: No such file or directory
 ls: a: No such file or directory
 ls: s: No such file or directory
 ls: d: No such file or directory
 ls: f: No such file or directory
 ls: a: No such file or directory
 ls: s: No such file or directory
 ls: f: No such file or directory
 ls: d: No such file or directory
 ls: /store: No such file or directory
 
 False
     """
     _, _, ret = runEOSCommand( path, 'ls')
     return ret == 0
 
 #also define an alias for backwards compatibility

def eostools.isFile ( path )

Returns True if a path is a file.

Tests for file existence.
Returns False for directories.
Works on EOS and local paths.

???This function works with local files, so not the same as isDirectory...
isFile and isDirectory should behave the same.

Definition at line 274 of file eostools.py.

 def isFile(path):
     """Returns True if a path is a file.
 
     Tests for file existence.
     Returns False for directories.
     Works on EOS and local paths.
     
     ???This function works with local files, so not the same as isDirectory...
     isFile and isDirectory should behave the same.
     """
 
     if not path.startswith('/eos') and not path.startswith('/store'):
         if( os.path.isfile(path) ):
             return True
         else:
             return False
     else: 
         out, _, _ = runXRDCommand(path,'existfile')
         return 'The file exists' in out
 

def eostools.isLFN ( path )

Tests whether this path is a CMS LFN (name starts with /store...)

Definition at line 57 of file eostools.py.

Referenced by lfnToEOS().

 def isLFN( path ):
     """Tests whether this path is a CMS LFN (name starts with /store...)"""
     # return re.match('^/store.*', path ) is not None
     return path.startswith('/store')
 

def eostools.lfnToEOS ( path )

Converts LFN to EOS.

If path is not an LFN in the first place, return path.
??? shouldn't we raise an exception?

Definition at line 108 of file eostools.py.

References isLFN().

Referenced by cat().

 def lfnToEOS( path ):
     """Converts LFN to EOS.
 
     If path is not an LFN in the first place, return path.
     ??? shouldn't we raise an exception?"""
     if isLFN(path):
         pfn = 'root://eoscms.cern.ch//eos/cms/' + path
         return pfn.replace('//store','/store') 
     else:
         return path
 
 #also define an alias for backwards compatibility

def eostools.lfnToPFN	(	path,
		tfcProt = `'rfio'`
	)

Converts an LFN to a PFN. For example:
/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21
->
root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms

This function only checks path, and does not access the storage system.
If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used.
Otherwise, is uses the default one. 

??? what is tfcprot?

Definition at line 77 of file eostools.py.

References str.

Referenced by isEOSDir(), runEOSCommand(), and runXRDCommand().

 def lfnToPFN( path, tfcProt = 'rfio'):
     """Converts an LFN to a PFN. For example:
     /store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21
     ->
     root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms
 
     This function only checks path, and does not access the storage system.
     If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used.
     Otherwise, is uses the default one. 
     
     ??? what is tfcprot? """
 
     if path.startswith("/store/"):
         path = path.replace("/store/","root://eoscms.cern.ch//eos/cms/store/")
     if path.startswith("/pnfs/psi.ch/cms/trivcat/"):
         path = path.replace("/pnfs/psi.ch/cms/trivcat/","root://t3se01.psi.ch//")
     #print "path to cmsFile():", path
     entity = cmsIO.cmsFile( path, tfcProt )
 #    tokens = cmsIO.splitPFN(entity.pfn)
     pfn = '%s://%s//%s/' % (entity.protocol,entity.host,entity.path)
     
     pfn = entity.pfn
     if tfcProt == 'rfio' and \
         entity.path.startswith("/eos/cms/") and \
                 str(entity.stat()).startswith("Error 3011: Unable to stat"):
 
             pfn.replace("/eos/cms","/castor/cern.ch/cms")
             pfn.replace("eoscms","castorcms")
     return pfn
 
 

def eostools.listFiles	(	path,
		rec = `False`,
		full_info = `False`
	)

Provides a list of the specified directory

Definition at line 300 of file eostools.py.

 def listFiles(path, rec = False, full_info = False):
     """Provides a list of the specified directory
     """
     # -- listing on the local filesystem --
     if os.path.isdir( path ):
         if not rec:
             # not recursive
             return [ '/'.join([path,file]) for file in os.listdir( path )]
         else:
             # recursive, directories are put in the list first,
             # followed by the list of all files in the directory tree
             result = []
             allFiles = []
             for root,dirs,files in os.walk(path):
                 result.extend( [ '/'.join([root,dir]) for dir in dirs] )
                 allFiles.extend( [ '/'.join([root,file]) for file in files] )
             result.extend(allFiles)
             return result
     # -- listing on EOS --
     cmd = 'dirlist'
     if rec:
         cmd = 'dirlistrec'
     files, _, _ = runXRDCommand(path, cmd)
     result = []
     for line in files.split('\n'):
         tokens = [t for t in line.split() if t]
         if tokens:
             #convert to an LFN
             # result.append(tuple(tokens))
             #COLIN need same interface for eos and local fs
             if full_info:
                 result.append( tokens)
             else:
                 result.append( tokens[4] )
     return result
 

def eostools.ls	(	path,
		rec = `False`
	)

Provides a simple list of the specified directory, works on EOS and locally

Definition at line 349 of file eostools.py.

Referenced by FWPFCandidateDetailView.addClusters(), HcalLaserEventFilter2012.addEventString(), FWPFCandidateDetailView.addHits(), METplusTrackMonitor.analyze(), DiDispStaMuonMonitor.analyze(), ObjMonitor.analyze(), DisplayGeom.analyze(), NoBPTXMonitor.analyze(), TopMonitor.analyze(), PhotonMonitor.analyze(), MuonMonitor.analyze(), JetMonitor.analyze(), METMonitor.analyze(), HTMonitor.analyze(), ZCounting.analyzeElectrons(), PixelLumiDQM.beginLuminosityBlock(), DQMProvInfo.blankAllLumiSections(), DQMProvInfo.blankPreviousLumiSections(), FWBeamSpotProxyBuilder.build(), FWPFClusterRPZProxyBuilder.build(), FWSiPixelClusterProxyBuilder.build(), FWPFCandidateDetailView.buildGLEventScene(), FWPFClusterRPZUtils.buildRhoPhiClusterLineSet(), FWPFClusterRPZUtils.buildRhoZClusterLineSet(), evf::EvFDaqDirector.bumpFile(), L1TSync.certifyLSBlock(), GenericTripletGenerator.clear(), CompositeLogicalTrajectoryFilter.CompositeLogicalTrajectoryFilter(), L1TSync.doFractionInSync(), PSMonitor.dqmAnalyze(), TriggerBxVsOrbitMonitor.dqmAnalyze(), LumiMonitor.dqmAnalyze(), TriggerBxMonitor.dqmAnalyze(), DQMDcsInfoClient.dqmEndJob(), L1TRate.endLuminosityBlock(), PixelLumiDQM.endLuminosityBlock(), RawEventFileWriterForBU.endOfLS(), FedRawDataInputSource.exceptionState(), RateBuffer.fill(), FastTimerService::PlotsPerJob.fill(), EcalDeadCellTriggerPrimitiveFilter.filter(), evf::EvFDaqDirector.findCurrentRunDir(), RawEventFileWriterForBU.finishFileWrite(), RateBuffer.getLSAlgoRate(), RateBuffer.getLSRate(), RateBuffer.getLSTechRate(), BeamSpotWorkflow.getNewRunList(), evf::EvFDaqDirector.getNextFromFileBroker(), L1TriggerJSONMonitoring.globalEndLuminosityBlockSummary(), HLTriggerJSONMonitoring.globalEndLuminosityBlockSummary(), BeamSpotRcdReader::theBSfromDB.init(), IdealZDCTrapezoid.initCorners(), IdealCastorTrapezoid.initCorners(), FWBeamSpotProxyBuilder.localModelChanges(), ls_cert_type(), lsbs_cert(), LumiProducerFromBrilcalc.LumiProducerFromBrilcalc(), copyFromCastor.main(), copyAndRename.main(), rename.main(), copyFiles.main(), main(), splitter.main(), checkRuns.main(), nlumis(), SeedFinderSelector.pass(), evf::EvFDaqDirector.preGlobalEndLumi(), AlpgenSource.readAlpgenEvent(), FedRawDataInputSource.readSupervisor(), evf::FastMonitoringService.reportLockWait(), lumi::TRGScalers2DB.retrieveData(), GenericTripletGenerator.run(), GenericPairGenerator.run(), BeamHaloPairGenerator.run(), fffnaming.runLumiPrefix(), fffnaming.runLumiPrefixFill(), evf::FastMonitoringService.setInStateSup(), MonitorElement.setLumi(), jsoncollector::FastMonitor.setNStreams(), DIPLumiDetail.setOrigin(), DIPLumiSummary.setOrigin(), FWPFClusterRPZProxyBuilder.sharedBuild(), RawEventFileWriterForBU.start(), TT6CMNSubtractor.subtract_(), IteratedMedianCMNSubtractor.subtract_(), SimpleCosmicBONSeeder.triplets(), evf::EvFDaqDirector.updateFuLock(), DTTimeEvolutionHisto.updateTimeSlot(), RawEventOutputModuleForBU< Consumer >.write(), DIPLumiDetail.~DIPLumiDetail(), and DIPLumiSummary.~DIPLumiSummary().

 def ls(path, rec = False):
     """Provides a simple list of the specified directory, works on EOS and locally"""
     return [eosToLFN(t) for t in listFiles(path, rec)]
 

def eostools.ls_EOS	(	path,
		rec = `False`
	)

Provides a simple list of the specified directory, works on EOS only, but is faster than the xrd version

Definition at line 353 of file eostools.py.

Referenced by datasetNotEmpty().

 def ls_EOS(path, rec = False):
     """Provides a simple list of the specified directory, works on EOS only, but is faster than the xrd version"""
     if rec:
         stdout, _, ret = runEOSCommand(path,'find','-f')
         return [eosToLFN(line) for line in stdout.split('\n') if line]
     else:
         stdout, _, ret = runEOSCommand(path,'ls')
         lfn = eosToLFN(path)
         return [os.path.join(lfn,line) for line in stdout.split('\n') if line]
 

def eostools.matchingFiles	(	path,
		regexp
	)

Return a list of files matching a regexp

Definition at line 519 of file eostools.py.

 def matchingFiles( path, regexp):
     """Return a list of files matching a regexp"""
 
     # print path, regexp
     pattern = re.compile( regexp )
     #files = ls_EOS(path)
     files = ls(path)
     # print files
     return [f for f in files if pattern.match(os.path.basename(f)) is not None]
 

def eostools.mkdir ( path )

Create a directory, either on EOS or locally

Definition at line 251 of file eostools.py.

Referenced by L1GtVhdlWriter.analyze(), spu.create_dir(), dd_to_html(), MillePedeMonitor.init(), evf::EvFDaqDirector.initRun(), LaserSorter.LaserSorter(), TFileDirectory.make(), L1GtVhdlWriterCore.makeFirmware(), pos::PixelConfigFile.makeNewVersion(), L1MuGlobalMuonTrigger.produce(), EgammaObjects.saveHistos(), VariablePlotter.setDir(), LaserSorter.streamFileName(), WatcherStreamFileReader.WatcherStreamFileReader(), and RawEventFileWriterForBU.writeJsds().

 def mkdir(path):
     """Create a directory, either on EOS or locally"""
     # print 'mkdir', path
     if isEOS( path ) or isLFN(path):
         createEOSDir(path)
     else:
         # recursive directory creation (like mkdir -p)
         os.makedirs(path)
     return path
 

def eostools.move	(	src,
		dest
	)

Move filename1 to filename2 locally to the same server

Definition at line 511 of file eostools.py.

 def move(src, dest):
     """Move filename1 to filename2 locally to the same server"""
     
     src = eosToLFN(src)
     dest = eosToLFN(dest)
 
     runXRDCommand(src,'mv', lfnToEOS(dest))
                 

def eostools.remove	(	files,
		rec = `False`
	)

Remove a list of files and directories, possibly recursively

Colin: Is that obsolete? why not use rm?

Definition at line 382 of file eostools.py.

 def remove( files, rec = False):
     """Remove a list of files and directories, possibly recursively
 
     Colin: Is that obsolete? why not use rm?"""
     for path in files:
         lfn = eosToLFN(path)
         if not rec:
             rm(path)
         else:
             #this should be used with care
             file_list = ls(path, rec = True)
             file_list.append(lfn)
             
             #order the files in depth order - i.e. remove the deepest files first
             files_rec = sorted([(len([ff for ff in f.split('/') if ff]), f) for f in file_list if f and f.startswith(lfn)], reverse = True)
             
             for f in files_rec:
                 rm(f[1])
                 

def eostools.rm	(	path,
		rec = `False`
	)

rm, works on EOS and locally.

Colin: should implement a -f mode and a confirmation when deleting dirs recursively.

Definition at line 363 of file eostools.py.

Referenced by remove().

 def rm(path, rec=False):
     """rm, works on EOS and locally.
 
     Colin: should implement a -f mode and a confirmation when deleting dirs recursively."""
     # print 'rm ', path
     path = lfnToEOS(path)
     if isEOS(path):
         if rec:
             runEOSCommand(path, 'rm', '-r')
         else: 
             runEOSCommand(path,'rm')
     elif os.path.exists(path):
         if not rec:
             os.remove( path )
         else:
             shutil.rmtree(path)
     else:
         raise ValueError(path + ' is not EOS and not local... should not happen!')
 

def eostools.runEOSCommand	(	path,
		cmd,
		args
	)

Run an eos command.

!!! Will, when the EOS command fails, it passes silently...
I think we should really try and raise an exception in case of problems.
should be possible as the return code is provided in the tuple returned by runner.

Definition at line 39 of file eostools.py.

References eosToLFN(), and lfnToPFN().

Referenced by chmod(), createEOSDir(), eosDirSize(), ls_EOS(), and rm().

 def runEOSCommand(path, cmd, *args):
     """Run an eos command.
 
     !!! Will, when the EOS command fails, it passes silently...
     I think we should really try and raise an exception in case of problems.
     should be possible as the return code is provided in the tuple returned by runner."""
     
     lfn = eosToLFN(path)
     pfn = lfnToPFN(lfn)
     tokens = cmsIO.splitPFN(pfn)
     
     #obviously, this is not nice
     command = [eos_select, cmd]
     command.extend(args)
     command.append(tokens[2])
     runner = cmsIO.cmsFileManip()
     return runner.runCommand(command)
 

def eostools.runXRDCommand	(	path,
		cmd,
		args
	)

Run an xrd command.

!!! Will, what is happening in case of problem?
??? At some point, should return a list of lines instead of a string.

Definition at line 23 of file eostools.py.

References eosToLFN(), and lfnToPFN().

Referenced by cat(), isDirectory(), isFile(), listFiles(), and move().

 def runXRDCommand(path, cmd, *args):
     """Run an xrd command.
 
     !!! Will, what is happening in case of problem?
     ??? At some point, should return a list of lines instead of a string."""
     
     lfn = eosToLFN(path)
     #print "lfn:", lfn, cmd
     tokens = cmsIO.splitPFN(lfnToPFN(lfn))
     
     command = ['xrd', tokens[1], cmd, tokens[2]]
     command.extend(args)
     runner = cmsIO.cmsFileManip()
     # print ' '.join(command)
     return runner.runCommand(command)
 

def eostools.setCAFPath ( )

Hack to get the CAF scripts on the PYTHONPATH

Definition at line 14 of file eostools.py.

 def setCAFPath():
     """Hack to get the CAF scripts on the PYTHONPATH"""
     caf = '/afs/cern.ch/cms/caf/python'
 
     if caf not in sys.path:
         sys.path.append(caf)
 setCAFPath()

def eostools.which ( cmd )

Definition at line 336 of file eostools.py.

Referenced by OOTPileupCorrData.apply(), MuonResidualsFitter.computeHistogramRangeAndBinning(), gen::EvtGenInterface.decay(), pat::PATSingleVertexSelector.filter_(), ConstantStepOdeSolver.getCoordinate(), OOTPileupCorrData.getCorrectionByID(), HBHENegativeEFilter.getEtaIndex(), LMFDat.getKeyList(), LMFDat.getNeighbour(), AbsElectronicODERHS.getParameter(), ConstantStepOdeSolver.getPeakTime(), HFQIE10Info.getRaw(), ConstantStepOdeSolver.integrateCoordinate(), ConstantStepOdeSolver.interpolateCoordinate(), MuonResidualsFitter.loglikelihood(), MuonResidualsTwoBin.mean(), PiecewiseScalingPolynomial.operator()(), QcdLowPtDQM.print(), MuonResidualsTwoBin.read(), InputTagDistributorService.retrieve(), MuonResidualsFitter.selectPeakResiduals(), AbsElectronicODERHS.setParameter(), MuonResidualsTwoBin.stdev(), ConstantStepOdeSolver.truncateCoordinate(), MuonResidualsTwoBin.wmean(), MuonResidualsTwoBin.write(), MuonResidualsFitter.write(), and ConstantStepOdeSolver.writeHistory().

 def which(cmd):
     command = ['which', cmd]
     runner = cmsIO.cmsFileManip()
     out, _, _ = runner.runCommand(command)
     
     lines = [line for line in out.split('\n') if line]
     if len(lines) == 1:
         return lines[0]
     elif len(lines) == 2:
         return lines[1]
     else:
         return lines
 

def eostools.xrdcp	(	src,
		dest
	)

Does a copy of files using xrd.

Colin: implement a generic cp interface as done for rm, ls, etc?

Definition at line 430 of file eostools.py.

 def xrdcp(src, dest):
     """Does a copy of files using xrd.
 
     Colin: implement a generic cp interface as done for rm, ls, etc?"""
     
     recursive = False
     
     #first the src file
     pfn_src = src
     if os.path.exists(src):
         #local
         pfn_src = src
         if os.path.isdir(src):
             recursive = True
     elif fileExists(src):
         src = eosToLFN(src)
         pfn_src = lfnToPFN(src)
         if isDirectory(src):
             recursive = True
     else:
         raise ValueError(src + ' does not exist.')
             
     #now the dest
     pfn_dest = dest
     if isEOSDir(dest):
         dest = eosToLFN(dest)
         pfn_dest = lfnToPFN(dest)
         if isDirectory(dest):
             tokens = cmsIO.splitPFN(pfn_dest)
             pfn_dest = '%s://%s//%s/' % (tokens[0],tokens[1],tokens[2])
     elif os.path.exists(dest):
         pfn_dest = dest
 
     command = ['xrdcp']
     if recursive:
         # print 'recursive'
         topDir = src.rstrip('/').split('/')[-1]
         if topDir != '.':
             dest = '/'.join([dest, topDir])
             # print 'mkdir ' + dest
             mkdir( dest )
         files = listFiles(src, rec=True)
         # pprint.pprint( [file[4] for file in files] )
         for srcFile in files:
             # srcFile = file[4]
             pfnSrcFile = srcFile
             if isEOSDir(srcFile):
                 srcFile = eosToLFN(srcFile)
                 pfnSrcFile = lfnToPFN(srcFile)
             destFile = srcFile.replace( src, '' )
             destFile = '/'.join([dest,destFile])
             pfnDestFile = destFile
             if isEOSDir(destFile):
                 lfnDestFile = eosToLFN(destFile)
                 pfnDestFile = lfnToPFN(lfnDestFile)
             # print 'srcFile', pfnSrcFile
             # print 'destFile', pfnDestFile
             if isFile(srcFile):
                 _xrdcpSingleFile(  pfnSrcFile, pfnDestFile )
             else:
                 mkdir(destFile)
     else:
         _xrdcpSingleFile( pfn_src, pfn_dest )
 
 

Variable Documentation

eostools.castorToLFN

Definition at line 75 of file eostools.py.

eostools.createCastorDir

Definition at line 249 of file eostools.py.

eostools.eos_select

Definition at line 12 of file eostools.py.

eostools.isCastorDir

Definition at line 150 of file eostools.py.

eostools.isCastorFile

Definition at line 195 of file eostools.py.

eostools.lfnToCastor

Definition at line 120 of file eostools.py.

Functions

Variables

Detailed Description

Function Documentation

Variable Documentation