Functions
def	_xrdcpSingleFile (pfn_src, pfn_dest)

def	cat (path)

def	chmod (path, mode)

def	cmsStage (absDestDir, files, force)

def	createEOSDir (path)

def	datasetNotEmpty (path, regexp)

def	eosDirSize (path)

def	eosToLFN (path)

def	fileExists (path)

def	isDirectory (path)

def	isEOS (path)

def	isEOSDir (path)

def	isEOSFile (path, tfcProt='rfio')

def	isFile (path)

def	isLFN (path)

def	lfnToEOS (path)

def	lfnToPFN (path, tfcProt='rfio')

def	listFiles (path, rec=False, full_info=False)

def	ls (path, rec=False)

def	ls_EOS (path, rec=False)

def	matchingFiles (path, regexp)

def	mkdir (path)

def	move (src, dest)

def	remove (files, rec=False)

def	rm (path, rec=False)

def	runEOSCommand (path, cmd, *args)

def	runXRDCommand (path, cmd, *args)

def	setCAFPath ()

def	which (cmd)

def	xrdcp (src, dest)

Variables
	castorToLFN

	createCastorDir

	eos_select

	isCastorDir

	isCastorFile

	lfnToCastor

Detailed Description

A module to manipulate files on EOS or on the local file system. Intended to have the same interface as castortools.py.

Function Documentation

◆ _xrdcpSingleFile()

def eostools._xrdcpSingleFile	(	pfn_src,
		pfn_dest
	)

private

Copies a single file using xrd.

Definition at line 495 of file eostools.py.

 def _xrdcpSingleFile( pfn_src, pfn_dest):
     """Copies a single file using xrd."""
     
     command = ['xrdcp']
     command.append(pfn_src)
     command.append(pfn_dest)
     # print ' '.join(command)
     run = True
     if run: 
         runner = cmsIO.cmsFileManip()
         out, err, ret = runner.runCommand(command)
         if err:
             print(out, file=sys.stderr)
             print(err, file=sys.stderr)
         return ret
  

References print().

Referenced by xrdcp().

◆ cat()

def eostools.cat ( path )

cat, works on EOS and locally

Definition at line 401 of file eostools.py.

 def cat(path):
     """cat, works on EOS and locally"""
     path = lfnToEOS(path)
     if isEOS(path):
         #print "the file to cat is:", path
         out, err, _ = runXRDCommand(path,'cat') 
         lines = []
         if out:
             pattern = re.compile('cat returned [0-9]+')
             for line in out.split('\n'):
                 match = pattern.search(line)
                 if line and match is not None:
                     lines.append(line.replace(match.group(0),''))
                     break
                 else:
                     lines.append(line)
         if err:
             print(out, file=sys.stderr)
             print(err, file=sys.stderr)
         allLines = '\n'.join(lines)
         if allLines and not allLines.endswith('\n'):
             allLines += '\n'
         return allLines
     else:
         content = file(path).read()
         if content and not content.endswith('\n'):
             content += '\n'
         return content
     

References FrontierConditions_GlobalTag_cff.file, isEOS(), join(), lfnToEOS(), print(), readEcalDQMStatus.read, and runXRDCommand().

Referenced by EnergyScaleCorrection.addScale(), EnergyScaleCorrection_class.AddScale(), EnergyScaleCorrection.addSmearing(), EnergyScaleCorrection_class.AddSmearing(), MVAJetTagPlotter.analyzeTag(), tadqm::TrackAnalyzer.bookHistosForEfficiencyFromHitPatter(), PFClusterFromHGCalMultiCluster.buildClusters(), CSCStripAmpResponse.calculateAmpResponse(), HGCalTriggerClusterIdentificationBDT.category(), CutBasedElectronID.cicSelection(), CutBasedElectronID.classify(), Rivet::HiggsTemplateCrossSections.classifyEvent(), DDLogicalPart.DDLogicalPart(), HGCalTriggerClusterIdentificationBDT.decision(), Rivet::HiggsTemplateCrossSections.error(), fwlite::Scanner< Collection >.fillDataSet(), HTXSFilter.filter(), edm::service::ELstatistics.formSummary(), GctFormatTranslateBase.gctJets(), HGCalTriggerClusterIdentificationBDT.initialize(), edm::service::MessageLoggerDefaults.limit(), edm::service::ELstatistics.noteGroupedCategory(), PhoMVACut.operator()(), GsfEleMVACut.operator()(), DDLLogicalPart.processElement(), MVAValueMapProducer< ParticleType >.produce(), HTXSRivetProducer.produce(), AlgorithmCalibration< T, CO >.readCategories(), edm::service::MessageLoggerDefaults.reportEvery(), HTXS.Rivet2Root(), edm::service::MessageLoggerDefaults.sev_limit(), edm::service::MessageLoggerDefaults.sev_reportEvery(), edm::service::MessageLoggerDefaults.sev_timespan(), edm::service::ELstatistics.summaryForJobReport(), TagProbeFitTreeAnalyzer.TagProbeFitTreeAnalyzer(), edm::service::MessageLoggerDefaults.timespan(), and HGCalTriggerClusterIdentificationBDT.value().

◆ chmod()

def eostools.chmod	(	path,
		mode
	)

Does chmod on a file or directory

Definition at line 294 of file eostools.py.

 def chmod(path, mode):
     """Does chmod on a file or directory"""
     #
     return runEOSCommand(path, 'chmod', '-r', str(mode))
  
  

References runEOSCommand(), and str.

Referenced by evf::EvFDaqDirector.initRun(), L1GtVhdlWriterCore.makeFirmware(), evf::EvFDaqDirector.openFULockfileStream(), L1GtVhdlWriterCore.writeAlgoSetup(), L1GtVhdlWriterCore.writeCond2intMap2File(), L1GtVhdlWriterCore.writeCondChipPkg(), L1GtVhdlWriterCore.writeConditionChipSetup(), L1GtVhdlWriterCore.writeDefValPkg(), L1GtVhdlWriterCore.writeEtmSetup(), L1GtVhdlWriterCore.writeMuonSetupVhdl(), and L1GtVhdlWriterCore.writeQsfSetupFiles().

◆ cmsStage()

def eostools.cmsStage	(	absDestDir,
		files,
		force
	)

Runs cmsStage with LFNs if possible

Definition at line 538 of file eostools.py.

 def cmsStage( absDestDir, files, force):
     """Runs cmsStage with LFNs if possible"""
  
     destIsEOSDir = isEOSDir(absDestDir)
     if destIsEOSDir: 
         createEOSDir( absDestDir )
  
     for fname in files:
         command = ['cmsStage']
         if force:
             command.append('-f')
         command.append(eosToLFN(fname))
         command.append(eosToLFN(absDestDir))
         print(' '.join(command))
         runner = cmsIO.cmsFileManip()
         runner.runCommand(command)

References createEOSDir(), eosToLFN(), isEOSDir(), join(), and print().

◆ createEOSDir()

def eostools.createEOSDir ( path )

Makes a directory in EOS

???Will, I'm quite worried by the fact that if this path already exists, and is
a file, everything will 'work'. But then we have a file, and not a directory,
while we expect a dir...

Definition at line 230 of file eostools.py.

 def createEOSDir( path ):
     """Makes a directory in EOS
  
     ???Will, I'm quite worried by the fact that if this path already exists, and is
     a file, everything will 'work'. But then we have a file, and not a directory,
     while we expect a dir..."""
     lfn = eosToLFN(path)
     if not isEOSFile(lfn):
     # if not isDirectory(lfn):
         runEOSCommand(lfn,'mkdir','-p')
         #        entity = cmsIO.cmsFile( lfn,"stageout")
         #        entity.mkdir([])
         #        # print 'created ', path
     if isDirectory(path):
         return path
     else:
         raise OSError('cannot create directory '+ path)
  
 #also define an alias for backwards compatibility

References eosToLFN(), isDirectory(), isEOSFile(), and runEOSCommand().

Referenced by cmsStage(), and mkdir().

◆ datasetNotEmpty()

def eostools.datasetNotEmpty	(	path,
		regexp
	)

Definition at line 529 of file eostools.py.

 def datasetNotEmpty( path, regexp ):
     pattern = re.compile( regexp )
     files = ls_EOS(path)
     
     for f in files:
         if pattern.match( os.path.basename(f) ) is not None:
             return 1
     return 0
     

References ls_EOS().

◆ eosDirSize()

def eostools.eosDirSize ( path )

Returns the size of a directory on EOS in GB.

Definition at line 216 of file eostools.py.

 def eosDirSize(path):
     '''Returns the size of a directory on EOS in GB.'''
     lfn = eosToLFN(path)
     res = runEOSCommand(lfn, 'find', '--size')
     output = res[0].split('\n')
     size = 0
     for file in output:
         try:
             size += float(file.split('=')[2])
         except IndexError:
             pass
     return size/1024/1024/1024
  
  

References eosToLFN(), dqmMemoryStats.float, runEOSCommand(), and submitPVValidationJobs.split().

◆ eosToLFN()

def eostools.eosToLFN ( path )

Converts a EOS PFN to an LFN.

Just strip out /eos/cms from path.
If this string is not found, return path.
??? Shouldn't we raise an exception instead?

Definition at line 66 of file eostools.py.

 def eosToLFN( path ):
     """Converts a EOS PFN to an LFN.
  
     Just strip out /eos/cms from path.
     If this string is not found, return path.
     ??? Shouldn't we raise an exception instead?"""
     return path.replace('root://eoscms.cern.ch/', '').replace('/eos/cms','')
  
 #also define an alias for backwards compatibility

References python.rootplot.root2matplotlib.replace().

Referenced by cmsStage(), createEOSDir(), eosDirSize(), isEOSDir(), ls(), ls_EOS(), move(), remove(), cmsBatch.rootfiles_to_eos_script(), runEOSCommand(), runXRDCommand(), and xrdcp().

◆ fileExists()

def eostools.fileExists ( path )

Returns true if path is a file or directory stored locally, or on EOS.

This function checks for the file or directory existence.

Definition at line 198 of file eostools.py.

 def fileExists( path ):
     """Returns true if path is a file or directory stored locally, or on EOS.
  
     This function checks for the file or directory existence."""
  
     eos = isEOSDir(path)
     result = False
     if eos:
         # print 'eos', path
         result = isEOSFile(path)
     else:
         # print 'not eos', path
         #check locally
         result = os.path.exists(path)
     # print result
     return result
  
  

References isEOSDir(), and isEOSFile().

Referenced by xrdcp().

◆ isDirectory()

def eostools.isDirectory ( path )

Returns True if path is a directory on EOS.

Tests for file existence. 
This function returns False for EOS files, and crashes with local paths

???Will, this function also seems to work for paths like:
/eos/cms/...
??? I think that it should work also for local files, see isFile.

Definition at line 261 of file eostools.py.

 def isDirectory(path):
     """Returns True if path is a directory on EOS.
  
     Tests for file existence. 
     This function returns False for EOS files, and crashes with local paths
  
     ???Will, this function also seems to work for paths like:
     /eos/cms/...
     ??? I think that it should work also for local files, see isFile."""
  
     out, _, _ = runXRDCommand(path,'existdir')
     return 'The directory exists' in out
  

References runXRDCommand().

Referenced by createEOSDir(), and xrdcp().

◆ isEOS()

def eostools.isEOS ( path )

Tests whether this path is a CMS EOS (name starts with /eos...)

Definition at line 62 of file eostools.py.

 def isEOS( path ):
     """Tests whether this path is a CMS EOS (name starts with /eos...)"""
     return path.startswith('/eos') or path.startswith('root://eoscms.cern.ch//eos/cms')
  

Referenced by cat(), mkdir(), and rm().

◆ isEOSDir()

def eostools.isEOSDir ( path )

Returns True if path is either:
/store/...
or
/eos/cms/store/...
or
root://eoscms.cern.ch//eos/cms/

Otherwise, returns False.

WARNING!! This function does not check for path existence,
and returns true also for plain files.
!!! Will, is my summary correct?

Definition at line 122 of file eostools.py.

 def isEOSDir( path ):
     """Returns True if path is either:
     /store/...
     or
     /eos/cms/store/...
     or
     root://eoscms.cern.ch//eos/cms/
  
     Otherwise, returns False.
  
     WARNING!! This function does not check for path existence,
     and returns true also for plain files.
     !!! Will, is my summary correct? 
     """
     if os.path.exists( path ):
         # path does not exist
         # COLIN: I think this condition could be removed,
         # as it duplicates the following one. 
         return False
     if not path.startswith('/eos') and not path.startswith('/store') and not path.startswith('root://eoscms.cern.ch//eos/cms/'):
         # neither an EOS PFN or a LFN.
         return False
     # at this stage, we must have an EOS PFN or an LFN
     pfn = lfnToPFN(eosToLFN(path))
     tokens = cmsIO.splitPFN(pfn)
     return tokens and tokens[1].lower().startswith('eos')
  
 #also define an alias for backwards compatibility

References eosToLFN(), and lfnToPFN().

Referenced by cmsStage(), fileExists(), and xrdcp().

◆ isEOSFile()

def eostools.isEOSFile	(	path,
		tfcProt = `'rfio'`
	)

Returns True if path is a file or directory stored on EOS (checks for path existence)
??? This function does not behave well if passed a non EOS path...
returns lots of error messages like:
>>> eostools.isEOSFile('/store/asdfasfd')
Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2
ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory

ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory

False

Definition at line 153 of file eostools.py.

 def isEOSFile( path, tfcProt = 'rfio'):
     """Returns True if path is a file or directory stored on EOS (checks for path existence)
     ??? This function does not behave well if passed a non EOS path...
     returns lots of error messages like:
 >>> eostools.isEOSFile('/store/asdfasfd')
 Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2
 ls: s: No such file or directory
 ls: t: No such file or directory
 ls: o: No such file or directory
 ls: r: No such file or directory
 ls: e: No such file or directory
 ls: a: No such file or directory
 ls: s: No such file or directory
 ls: d: No such file or directory
 ls: f: No such file or directory
 ls: a: No such file or directory
 ls: s: No such file or directory
 ls: f: No such file or directory
 ls: d: No such file or directory
 ls: /store: No such file or directory
  
 ls: s: No such file or directory
 ls: t: No such file or directory
 ls: o: No such file or directory
 ls: r: No such file or directory
 ls: e: No such file or directory
 ls: a: No such file or directory
 ls: s: No such file or directory
 ls: d: No such file or directory
 ls: f: No such file or directory
 ls: a: No such file or directory
 ls: s: No such file or directory
 ls: f: No such file or directory
 ls: d: No such file or directory
 ls: /store: No such file or directory
  
 False
     """
     _, _, ret = runEOSCommand( path, 'ls')
     return ret == 0
  
 #also define an alias for backwards compatibility

References runEOSCommand().

Referenced by createEOSDir(), and fileExists().

◆ isFile()

def eostools.isFile ( path )

Returns True if a path is a file.

Tests for file existence.
Returns False for directories.
Works on EOS and local paths.

???This function works with local files, so not the same as isDirectory...
isFile and isDirectory should behave the same.

Definition at line 274 of file eostools.py.

 def isFile(path):
     """Returns True if a path is a file.
  
     Tests for file existence.
     Returns False for directories.
     Works on EOS and local paths.
     
     ???This function works with local files, so not the same as isDirectory...
     isFile and isDirectory should behave the same.
     """
  
     if not path.startswith('/eos') and not path.startswith('/store'):
         if( os.path.isfile(path) ):
             return True
         else:
             return False
     else: 
         out, _, _ = runXRDCommand(path,'existfile')
         return 'The file exists' in out
  

References if(), and runXRDCommand().

Referenced by xrdcp().

◆ isLFN()

def eostools.isLFN ( path )

Tests whether this path is a CMS LFN (name starts with /store...)

Definition at line 57 of file eostools.py.

 def isLFN( path ):
     """Tests whether this path is a CMS LFN (name starts with /store...)"""
     # return re.match('^/store.*', path ) is not None
     return path.startswith('/store')
  

Referenced by lfnToEOS(), and mkdir().

◆ lfnToEOS()

def eostools.lfnToEOS ( path )

Converts LFN to EOS.

If path is not an LFN in the first place, return path.
??? shouldn't we raise an exception?

Definition at line 108 of file eostools.py.

 def lfnToEOS( path ):
     """Converts LFN to EOS.
  
     If path is not an LFN in the first place, return path.
     ??? shouldn't we raise an exception?"""
     if isLFN(path):
         pfn = 'root://eoscms.cern.ch//eos/cms/' + path
         return pfn.replace('//store','/store') 
     else:
         return path
  
 #also define an alias for backwards compatibility

References isLFN().

Referenced by cat(), move(), and rm().

◆ lfnToPFN()

def eostools.lfnToPFN	(	path,
		tfcProt = `'rfio'`
	)

Converts an LFN to a PFN. For example:
/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21
->
root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms

This function only checks path, and does not access the storage system.
If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used.
Otherwise, is uses the default one. 

??? what is tfcprot?

Definition at line 77 of file eostools.py.

 def lfnToPFN( path, tfcProt = 'rfio'):
     """Converts an LFN to a PFN. For example:
     /store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21
     ->
     root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms
  
     This function only checks path, and does not access the storage system.
     If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used.
     Otherwise, is uses the default one. 
     
     ??? what is tfcprot? """
  
     if path.startswith("/store/"):
         path = path.replace("/store/","root://eoscms.cern.ch//eos/cms/store/")
     if path.startswith("/pnfs/psi.ch/cms/trivcat/"):
         path = path.replace("/pnfs/psi.ch/cms/trivcat/","root://t3se01.psi.ch//")
     #print "path to cmsFile():", path
     entity = cmsIO.cmsFile( path, tfcProt )
 #    tokens = cmsIO.splitPFN(entity.pfn)
     pfn = '%s://%s//%s/' % (entity.protocol,entity.host,entity.path)
     
     pfn = entity.pfn
     if tfcProt == 'rfio' and \
         entity.path.startswith("/eos/cms/") and \
                 str(entity.stat()).startswith("Error 3011: Unable to stat"):
  
             pfn.replace("/eos/cms","/castor/cern.ch/cms")
             pfn.replace("eoscms","castorcms")
     return pfn
  
  

References str.

Referenced by isEOSDir(), runEOSCommand(), runXRDCommand(), and xrdcp().

◆ listFiles()

def eostools.listFiles	(	path,
		rec = `False`,
		full_info = `False`
	)

Provides a list of the specified directory

Definition at line 300 of file eostools.py.

 def listFiles(path, rec = False, full_info = False):
     """Provides a list of the specified directory
     """
     # -- listing on the local filesystem --
     if os.path.isdir( path ):
         if not rec:
             # not recursive
             return [ '/'.join([path,file]) for file in os.listdir( path )]
         else:
             # recursive, directories are put in the list first,
             # followed by the list of all files in the directory tree
             result = []
             allFiles = []
             for root,dirs,files in os.walk(path):
                 result.extend( [ '/'.join([root,dir]) for dir in dirs] )
                 allFiles.extend( [ '/'.join([root,file]) for file in files] )
             result.extend(allFiles)
             return result
     # -- listing on EOS --
     cmd = 'dirlist'
     if rec:
         cmd = 'dirlistrec'
     files, _, _ = runXRDCommand(path, cmd)
     result = []
     for line in files.split('\n'):
         tokens = [t for t in line.split() if t]
         if tokens:
             #convert to an LFN
             # result.append(tuple(tokens))
             #COLIN need same interface for eos and local fs
             if full_info:
                 result.append( tokens)
             else:
                 result.append( tokens[4] )
     return result
  

References join(), and runXRDCommand().

Referenced by ls(), and xrdcp().

◆ ls()

def eostools.ls	(	path,
		rec = `False`
	)

Provides a simple list of the specified directory, works on EOS and locally

Definition at line 349 of file eostools.py.

 def ls(path, rec = False):
     """Provides a simple list of the specified directory, works on EOS and locally"""
     return [eosToLFN(t) for t in listFiles(path, rec)]
  

References eosToLFN(), and listFiles().

◆ ls_EOS()

def eostools.ls_EOS	(	path,
		rec = `False`
	)

Provides a simple list of the specified directory, works on EOS only, but is faster than the xrd version

Definition at line 353 of file eostools.py.

 def ls_EOS(path, rec = False):
     """Provides a simple list of the specified directory, works on EOS only, but is faster than the xrd version"""
     if rec:
         stdout, _, ret = runEOSCommand(path,'find','-f')
         return [eosToLFN(line) for line in stdout.split('\n') if line]
     else:
         stdout, _, ret = runEOSCommand(path,'ls')
         lfn = eosToLFN(path)
         return [os.path.join(lfn,line) for line in stdout.split('\n') if line]
  

References eosToLFN(), and runEOSCommand().

Referenced by datasetNotEmpty().

◆ matchingFiles()

def eostools.matchingFiles	(	path,
		regexp
	)

Return a list of files matching a regexp

Definition at line 519 of file eostools.py.

 def matchingFiles( path, regexp):
     """Return a list of files matching a regexp"""
  
     # print path, regexp
     pattern = re.compile( regexp )
     #files = ls_EOS(path)
     files = ls(path)
     # print files
     return [f for f in files if pattern.match(os.path.basename(f)) is not None]
  

References ls().

◆ mkdir()

def eostools.mkdir ( path )

Create a directory, either on EOS or locally

Definition at line 251 of file eostools.py.

 def mkdir(path):
     """Create a directory, either on EOS or locally"""
     # print 'mkdir', path
     if isEOS( path ) or isLFN(path):
         createEOSDir(path)
     else:
         # recursive directory creation (like mkdir -p)
         os.makedirs(path)
     return path
  

References createEOSDir(), isEOS(), and isLFN().

Referenced by L1GtVhdlWriter.analyze(), spu.create_dir(), dd_to_html(), MillePedeMonitor.init(), evf::EvFDaqDirector.initRun(), LaserSorter.LaserSorter(), L1GtVhdlWriterCore.makeFirmware(), pos::PixelConfigFile.makeNewVersion(), L1MuGlobalMuonTrigger.produce(), EgammaObjects.saveHistos(), VariablePlotter.setDir(), AlignmentMonitorBase.startingNewLoop(), LaserSorter.streamFileName(), WatcherStreamFileReader.WatcherStreamFileReader(), DQMIO2histo.DQMIO.write_to_file(), RawEventFileWriterForBU.writeJsds(), and xrdcp().

◆ move()

def eostools.move	(	src,
		dest
	)

Move filename1 to filename2 locally to the same server

Definition at line 511 of file eostools.py.

 def move(src, dest):
     """Move filename1 to filename2 locally to the same server"""
     
     src = eosToLFN(src)
     dest = eosToLFN(dest)
  
     runXRDCommand(src,'mv', lfnToEOS(dest))
                 

References eosToLFN(), lfnToEOS(), and runXRDCommand().

◆ remove()

def eostools.remove	(	files,
		rec = `False`
	)

Remove a list of files and directories, possibly recursively

Colin: Is that obsolete? why not use rm?

Definition at line 382 of file eostools.py.

 def remove( files, rec = False):
     """Remove a list of files and directories, possibly recursively
  
     Colin: Is that obsolete? why not use rm?"""
     for path in files:
         lfn = eosToLFN(path)
         if not rec:
             rm(path)
         else:
             #this should be used with care
             file_list = ls(path, rec = True)
             file_list.append(lfn)
             
             #order the files in depth order - i.e. remove the deepest files first
             files_rec = sorted([(len([ff for ff in f.split('/') if ff]), f) for f in file_list if f and f.startswith(lfn)], reverse = True)
             
             for f in files_rec:
                 rm(f[1])
                 

References eosToLFN(), ls(), and rm().

◆ rm()

def eostools.rm	(	path,
		rec = `False`
	)

rm, works on EOS and locally.

Colin: should implement a -f mode and a confirmation when deleting dirs recursively.

Definition at line 363 of file eostools.py.

 def rm(path, rec=False):
     """rm, works on EOS and locally.
  
     Colin: should implement a -f mode and a confirmation when deleting dirs recursively."""
     # print 'rm ', path
     path = lfnToEOS(path)
     if isEOS(path):
         if rec:
             runEOSCommand(path, 'rm', '-r')
         else: 
             runEOSCommand(path,'rm')
     elif os.path.exists(path):
         if not rec:
             os.remove( path )
         else:
             shutil.rmtree(path)
     else:
         raise ValueError(path + ' is not EOS and not local... should not happen!')
  

References isEOS(), lfnToEOS(), and runEOSCommand().

Referenced by remove().

◆ runEOSCommand()

def eostools.runEOSCommand	(		path,
			cmd,
		*	args
	)

Run an eos command.

!!! Will, when the EOS command fails, it passes silently...
I think we should really try and raise an exception in case of problems.
should be possible as the return code is provided in the tuple returned by runner.

Definition at line 39 of file eostools.py.

 def runEOSCommand(path, cmd, *args):
     """Run an eos command.
  
     !!! Will, when the EOS command fails, it passes silently...
     I think we should really try and raise an exception in case of problems.
     should be possible as the return code is provided in the tuple returned by runner."""
     
     lfn = eosToLFN(path)
     pfn = lfnToPFN(lfn)
     tokens = cmsIO.splitPFN(pfn)
     
     #obviously, this is not nice
     command = [eos_select, cmd]
     command.extend(args)
     command.append(tokens[2])
     runner = cmsIO.cmsFileManip()
     return runner.runCommand(command)
  

References eosToLFN(), and lfnToPFN().

Referenced by chmod(), createEOSDir(), eosDirSize(), isEOSFile(), ls_EOS(), and rm().

◆ runXRDCommand()

def eostools.runXRDCommand	(		path,
			cmd,
		*	args
	)

Run an xrd command.

!!! Will, what is happening in case of problem?
??? At some point, should return a list of lines instead of a string.

Definition at line 23 of file eostools.py.

 def runXRDCommand(path, cmd, *args):
     """Run an xrd command.
  
     !!! Will, what is happening in case of problem?
     ??? At some point, should return a list of lines instead of a string."""
     
     lfn = eosToLFN(path)
     #print "lfn:", lfn, cmd
     tokens = cmsIO.splitPFN(lfnToPFN(lfn))
     
     command = ['xrd', tokens[1], cmd, tokens[2]]
     command.extend(args)
     runner = cmsIO.cmsFileManip()
     # print ' '.join(command)
     return runner.runCommand(command)
  

References eosToLFN(), and lfnToPFN().

Referenced by cat(), isDirectory(), isFile(), listFiles(), and move().

◆ setCAFPath()

def eostools.setCAFPath ( )

Hack to get the CAF scripts on the PYTHONPATH

Definition at line 14 of file eostools.py.

 def setCAFPath():
     """Hack to get the CAF scripts on the PYTHONPATH"""
     caf = '/afs/cern.ch/cms/caf/python'
  
     if caf not in sys.path:
         sys.path.append(caf)
 setCAFPath()

◆ which()

def eostools.which ( cmd )

Definition at line 336 of file eostools.py.

 def which(cmd):
     command = ['which', cmd]
     runner = cmsIO.cmsFileManip()
     out, _, _ = runner.runCommand(command)
     
     lines = [line for line in out.split('\n') if line]
     if len(lines) == 1:
         return lines[0]
     elif len(lines) == 2:
         return lines[1]
     else:
         return lines
  

◆ xrdcp()

def eostools.xrdcp	(	src,
		dest
	)

Does a copy of files using xrd.

Colin: implement a generic cp interface as done for rm, ls, etc?

Definition at line 430 of file eostools.py.

 def xrdcp(src, dest):
     """Does a copy of files using xrd.
  
     Colin: implement a generic cp interface as done for rm, ls, etc?"""
     
     recursive = False
     
     #first the src file
     pfn_src = src
     if os.path.exists(src):
         #local
         pfn_src = src
         if os.path.isdir(src):
             recursive = True
     elif fileExists(src):
         src = eosToLFN(src)
         pfn_src = lfnToPFN(src)
         if isDirectory(src):
             recursive = True
     else:
         raise ValueError(src + ' does not exist.')
             
     #now the dest
     pfn_dest = dest
     if isEOSDir(dest):
         dest = eosToLFN(dest)
         pfn_dest = lfnToPFN(dest)
         if isDirectory(dest):
             tokens = cmsIO.splitPFN(pfn_dest)
             pfn_dest = '%s://%s//%s/' % (tokens[0],tokens[1],tokens[2])
     elif os.path.exists(dest):
         pfn_dest = dest
  
     command = ['xrdcp']
     if recursive:
         # print 'recursive'
         topDir = src.rstrip('/').split('/')[-1]
         if topDir != '.':
             dest = '/'.join([dest, topDir])
             # print 'mkdir ' + dest
             mkdir( dest )
         files = listFiles(src, rec=True)
         # pprint.pprint( [file[4] for file in files] )
         for srcFile in files:
             # srcFile = file[4]
             pfnSrcFile = srcFile
             if isEOSDir(srcFile):
                 srcFile = eosToLFN(srcFile)
                 pfnSrcFile = lfnToPFN(srcFile)
             destFile = srcFile.replace( src, '' )
             destFile = '/'.join([dest,destFile])
             pfnDestFile = destFile
             if isEOSDir(destFile):
                 lfnDestFile = eosToLFN(destFile)
                 pfnDestFile = lfnToPFN(lfnDestFile)
             # print 'srcFile', pfnSrcFile
             # print 'destFile', pfnDestFile
             if isFile(srcFile):
                 _xrdcpSingleFile(  pfnSrcFile, pfnDestFile )
             else:
                 mkdir(destFile)
     else:
         _xrdcpSingleFile( pfn_src, pfn_dest )
  
  

References _xrdcpSingleFile(), eosToLFN(), fileExists(), isDirectory(), isEOSDir(), isFile(), join(), lfnToPFN(), listFiles(), mkdir(), and submitPVValidationJobs.split().

Variable Documentation

◆ castorToLFN

eostools.castorToLFN

Definition at line 75 of file eostools.py.

◆ createCastorDir

eostools.createCastorDir

Definition at line 249 of file eostools.py.

◆ eos_select

eostools.eos_select

Definition at line 12 of file eostools.py.

◆ isCastorDir

eostools.isCastorDir

Definition at line 150 of file eostools.py.

◆ isCastorFile

eostools.isCastorFile

Definition at line 195 of file eostools.py.

◆ lfnToCastor

eostools.lfnToCastor

Definition at line 120 of file eostools.py.

Functions

Variables

Detailed Description

Function Documentation

◆ _xrdcpSingleFile()

◆ cat()

◆ chmod()

◆ cmsStage()

◆ createEOSDir()

◆ datasetNotEmpty()

◆ eosDirSize()

◆ eosToLFN()

◆ fileExists()

◆ isDirectory()

◆ isEOS()

◆ isEOSDir()

◆ isEOSFile()

◆ isFile()

◆ isLFN()

◆ lfnToEOS()

◆ lfnToPFN()

◆ listFiles()

◆ ls()

◆ ls_EOS()

◆ matchingFiles()

◆ mkdir()

◆ move()

◆ remove()

◆ rm()

◆ runEOSCommand()

◆ runXRDCommand()

◆ setCAFPath()

◆ which()

◆ xrdcp()

Variable Documentation

◆ castorToLFN

◆ createCastorDir

◆ eos_select

◆ isCastorDir

◆ isCastorFile

◆ lfnToCastor