CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
eostools Namespace Reference

Functions

def _xrdcpSingleFile
 
def cat
 
def chmod
 
def cmsStage
 
def createEOSDir
 
def datasetNotEmpty
 
def eosDirSize
 
def eosToLFN
 
def fileExists
 
def isDirectory
 
def isEOS
 
def isEOSDir
 
def isEOSFile
 
def isFile
 
def isLFN
 
def lfnToEOS
 
def lfnToPFN
 
def listFiles
 
def ls
 
def ls_EOS
 
def matchingFiles
 
def mkdir
 
def move
 
def remove
 
def rm
 
def runEOSCommand
 
def runXRDCommand
 
def setCAFPath
 
def which
 
def xrdcp
 

Variables

 castorToLFN = eosToLFN
 
 createCastorDir = createEOSDir
 
string eos_select = '/afs/cern.ch/project/eos/installation/cms/bin/eos.select'
 
 isCastorDir = isEOSDir
 
 isCastorFile = isEOSFile
 
 lfnToCastor = lfnToEOS
 

Function Documentation

def eostools._xrdcpSingleFile (   pfn_src,
  pfn_dest 
)
private 
Copies a single file using xrd.

Definition at line 494 of file eostools.py.

494 
495 def _xrdcpSingleFile( pfn_src, pfn_dest):
496  """Copies a single file using xrd."""
497 
498  command = ['xrdcp']
499  command.append(pfn_src)
500  command.append(pfn_dest)
501  # print ' '.join(command)
502  run = True
503  if run:
504  runner = cmsIO.cmsFileManip()
505  out, err, ret = runner.runCommand(command)
506  if err:
507  print >> sys.stderr, out
508  print >> sys.stderr, err
509  return ret
eostools._xrdcpSingleFile
def _xrdcpSingleFile
Definition: eostools.py:494
def eostools.cat (   path) 
cat, works on EOS and locally

Definition at line 400 of file eostools.py.

Referenced by EnergyScaleCorrection_class.AddScale(), EnergyScaleCorrection_class.AddSmearing(), MVAJetTagPlotter.analyzeTag(), TrackAnalyzer.bookHistosForEfficiencyFromHitPatter(), CSCStripAmpResponse.calculateAmpResponse(), CutBasedElectronID.cicSelection(), CutBasedElectronID.classify(), fwlite::Scanner< Collection >.fillDataSet(), edm::service::ELstatistics.formSummary(), GsfEleMVACut.operator()(), PhoMVACut.operator()(), DDStreamer.parts_read(), DDLLogicalPart.processElement(), AlgorithmCalibration< T, CO >.readCategories(), edm::service::ELstatistics.summaryForJobReport(), and TagProbeFitTreeAnalyzer.TagProbeFitTreeAnalyzer().

400 
401 def cat(path):
402  """cat, works on EOS and locally"""
403  path = lfnToEOS(path)
404  if isEOS(path):
405  #print "the file to cat is:", path
406  out, err, _ = runXRDCommand(path,'cat')
407  lines = []
408  if out:
409  pattern = re.compile('cat returned [0-9]+')
410  for line in out.split('\n'):
411  match = pattern.search(line)
412  if line and match is not None:
413  lines.append(line.replace(match.group(0),''))
414  break
415  else:
416  lines.append(line)
417  if err:
418  print >> sys.stderr, out
419  print >> sys.stderr, err
420  allLines = '\n'.join(lines)
421  if allLines and not allLines.endswith('\n'):
422  allLines += '\n'
423  return allLines
424  else:
425  content = file(path).read()
426  if content and not content.endswith('\n'):
427  content += '\n'
428  return content
eostools.cat
def cat
Definition: eostools.py:400
eostools.lfnToEOS
def lfnToEOS
Definition: eostools.py:107
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
eostools.runXRDCommand
def runXRDCommand
Definition: eostools.py:22
eostools.isEOS
def isEOS
Definition: eostools.py:61
def eostools.chmod (   path,
  mode 
) 
Does chmod on a file or directory

Definition at line 293 of file eostools.py.

Referenced by evf::EvFDaqDirector.initRun(), L1GtVhdlWriterCore.makeFirmware(), evf::EvFDaqDirector.openFULockfileStream(), L1GtVhdlWriterCore.writeAlgoSetup(), L1GtVhdlWriterCore.writeCond2intMap2File(), L1GtVhdlWriterCore.writeCondChipPkg(), L1GtVhdlWriterCore.writeConditionChipSetup(), L1GtVhdlWriterCore.writeDefValPkg(), L1GtVhdlWriterCore.writeEtmSetup(), L1GtVhdlWriterCore.writeMuonSetupVhdl(), and L1GtVhdlWriterCore.writeQsfSetupFiles().

293 
294 def chmod(path, mode):
295  """Does chmod on a file or directory"""
296  #
297  return runEOSCommand(path, 'chmod', '-r', str(mode))
298 
eostools.chmod
def chmod
Definition: eostools.py:293
eostools.runEOSCommand
def runEOSCommand
Definition: eostools.py:38
def eostools.cmsStage (   absDestDir,
  files,
  force 
) 
Runs cmsStage with LFNs if possible

Definition at line 537 of file eostools.py.

537 
538 def cmsStage( absDestDir, files, force):
539  """Runs cmsStage with LFNs if possible"""
540 
541  destIsEOSDir = isEOSDir(absDestDir)
542  if destIsEOSDir:
543  createEOSDir( absDestDir )
544 
545  for fname in files:
546  command = ['cmsStage']
547  if force:
548  command.append('-f')
549  command.append(eosToLFN(fname))
550  command.append(eosToLFN(absDestDir))
551  print ' '.join(command)
552  runner = cmsIO.cmsFileManip()
553  runner.runCommand(command)
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.isEOSDir
def isEOSDir
Definition: eostools.py:121
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
eostools.cmsStage
def cmsStage
Definition: eostools.py:537
eostools.createEOSDir
def createEOSDir
Definition: eostools.py:229
def eostools.createEOSDir (   path) 
Makes a directory in EOS

???Will, I'm quite worried by the fact that if this path already exists, and is
a file, everything will 'work'. But then we have a file, and not a directory,
while we expect a dir...

Definition at line 229 of file eostools.py.

Referenced by cmsStage().

229 
230 def createEOSDir( path ):
231  """Makes a directory in EOS
232 
233  ???Will, I'm quite worried by the fact that if this path already exists, and is
234  a file, everything will 'work'. But then we have a file, and not a directory,
235  while we expect a dir..."""
236  lfn = eosToLFN(path)
237  if not isEOSFile(lfn):
238  # if not isDirectory(lfn):
239  runEOSCommand(lfn,'mkdir','-p')
240  # entity = cmsIO.cmsFile( lfn,"stageout")
241  # entity.mkdir([])
242  # # print 'created ', path
243  if isDirectory(path):
244  return path
245  else:
246  raise OSError('cannot create directory '+ path)
247 
#also define an alias for backwards compatibility
eostools.isEOSFile
def isEOSFile
Definition: eostools.py:152
eostools.isDirectory
def isDirectory
Definition: eostools.py:260
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.runEOSCommand
def runEOSCommand
Definition: eostools.py:38
eostools.createEOSDir
def createEOSDir
Definition: eostools.py:229
def eostools.datasetNotEmpty (   path,
  regexp 
)

Definition at line 528 of file eostools.py.

528 
529 def datasetNotEmpty( path, regexp ):
530  pattern = re.compile( regexp )
531  files = ls_EOS(path)
532 
533  for f in files:
534  if pattern.match( os.path.basename(f) ) is not None:
535  return 1
536  return 0
eostools.ls_EOS
def ls_EOS
Definition: eostools.py:352
eostools.datasetNotEmpty
def datasetNotEmpty
Definition: eostools.py:528
def eostools.eosDirSize (   path) 
Returns the size of a directory on EOS in GB.

Definition at line 215 of file eostools.py.

215 
216 def eosDirSize(path):
217  '''Returns the size of a directory on EOS in GB.'''
218  lfn = eosToLFN(path)
219  res = runEOSCommand(lfn, 'find', '--size')
220  output = res[0].split('\n')
221  size = 0
222  for file in output:
223  try:
224  size += float(file.split('=')[2])
225  except IndexError:
226  pass
227  return size/1024/1024/1024
228 
eostools.eosDirSize
def eosDirSize
Definition: eostools.py:215
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.runEOSCommand
def runEOSCommand
Definition: eostools.py:38
split
double split
Definition: MVATrainer.cc:139
def eostools.eosToLFN (   path) 
Converts a EOS PFN to an LFN.

Just strip out /eos/cms from path.
If this string is not found, return path.
??? Shouldn't we raise an exception instead?

Definition at line 65 of file eostools.py.

References python.rootplot.root2matplotlib.replace().

Referenced by cmsStage(), createEOSDir(), eosDirSize(), isEOSDir(), cmsBatch.rootfiles_to_eos_script(), runEOSCommand(), and runXRDCommand().

65 
66 def eosToLFN( path ):
67  """Converts a EOS PFN to an LFN.
68 
69  Just strip out /eos/cms from path.
70  If this string is not found, return path.
71  ??? Shouldn't we raise an exception instead?"""
72  return path.replace('root://eoscms.cern.ch/', '').replace('/eos/cms','')
73 
#also define an alias for backwards compatibility
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
def eostools.fileExists (   path) 
Returns true if path is a file or directory stored locally, or on EOS.

This function checks for the file or directory existence.

Definition at line 197 of file eostools.py.

197 
198 def fileExists( path ):
199  """Returns true if path is a file or directory stored locally, or on EOS.
200 
201  This function checks for the file or directory existence."""
202 
203  eos = isEOSDir(path)
204  result = False
205  if eos:
206  # print 'eos', path
207  result = isEOSFile(path)
208  else:
209  # print 'not eos', path
210  #check locally
211  result = os.path.exists(path)
212  # print result
213  return result
214 
eostools.isEOSFile
def isEOSFile
Definition: eostools.py:152
eostools.fileExists
def fileExists
Definition: eostools.py:197
eostools.isEOSDir
def isEOSDir
Definition: eostools.py:121
def eostools.isDirectory (   path) 
Returns True if path is a directory on EOS.

Tests for file existence. 
This function returns False for EOS files, and crashes with local paths

???Will, this function also seems to work for paths like:
/eos/cms/...
??? I think that it should work also for local files, see isFile.

Definition at line 260 of file eostools.py.

Referenced by createEOSDir().

260 
261 def isDirectory(path):
262  """Returns True if path is a directory on EOS.
263 
264  Tests for file existence.
265  This function returns False for EOS files, and crashes with local paths
266 
267  ???Will, this function also seems to work for paths like:
268  /eos/cms/...
269  ??? I think that it should work also for local files, see isFile."""
270 
271  out, _, _ = runXRDCommand(path,'existdir')
272  return 'The directory exists' in out
eostools.isDirectory
def isDirectory
Definition: eostools.py:260
eostools.runXRDCommand
def runXRDCommand
Definition: eostools.py:22
def eostools.isEOS (   path) 
Tests whether this path is a CMS EOS (name starts with /eos...)

Definition at line 61 of file eostools.py.

Referenced by cat().

61 
62 def isEOS( path ):
63  """Tests whether this path is a CMS EOS (name starts with /eos...)"""
64  return path.startswith('/eos') or path.startswith('root://eoscms.cern.ch//eos/cms')
eostools.isEOS
def isEOS
Definition: eostools.py:61
def eostools.isEOSDir (   path) 
Returns True if path is either:
/store/...
or
/eos/cms/store/...
or
root://eoscms.cern.ch//eos/cms/

Otherwise, returns False.

WARNING!! This function does not check for path existence,
and returns true also for plain files.
!!! Will, is my summary correct?

Definition at line 121 of file eostools.py.

References eosToLFN(), and lfnToPFN().

Referenced by cmsStage(), and fileExists().

121 
122 def isEOSDir( path ):
123  """Returns True if path is either:
124  /store/...
125  or
126  /eos/cms/store/...
127  or
128  root://eoscms.cern.ch//eos/cms/
129 
130  Otherwise, returns False.
131 
132  WARNING!! This function does not check for path existence,
133  and returns true also for plain files.
134  !!! Will, is my summary correct?
135  """
136  if os.path.exists( path ):
137  # path does not exist
138  # COLIN: I think this condition could be removed,
139  # as it duplicates the following one.
140  return False
141  if not path.startswith('/eos') and not path.startswith('/store') and not path.startswith('root://eoscms.cern.ch//eos/cms/'):
142  # neither an EOS PFN or a LFN.
143  return False
144  # at this stage, we must have an EOS PFN or an LFN
145  pfn = lfnToPFN(eosToLFN(path))
146  tokens = cmsIO.splitPFN(pfn)
147  return tokens and tokens[1].lower().startswith('eos')
148 
#also define an alias for backwards compatibility
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.isEOSDir
def isEOSDir
Definition: eostools.py:121
eostools.lfnToPFN
def lfnToPFN
Definition: eostools.py:76
def eostools.isEOSFile (   path,
  tfcProt = 'rfio' 
) 
Returns True if path is a file or directory stored on EOS (checks for path existence)
??? This function does not behave well if passed a non EOS path...
returns lots of error messages like:
>>> eostools.isEOSFile('/store/asdfasfd')
Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2
ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory

ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory

False

Definition at line 152 of file eostools.py.

Referenced by createEOSDir(), and fileExists().

152 
153 def isEOSFile( path, tfcProt = 'rfio'):
154  """Returns True if path is a file or directory stored on EOS (checks for path existence)
155  ??? This function does not behave well if passed a non EOS path...
156  returns lots of error messages like:
157 >>> eostools.isEOSFile('/store/asdfasfd')
158 Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2
159 ls: s: No such file or directory
160 ls: t: No such file or directory
161 ls: o: No such file or directory
162 ls: r: No such file or directory
163 ls: e: No such file or directory
164 ls: a: No such file or directory
165 ls: s: No such file or directory
166 ls: d: No such file or directory
167 ls: f: No such file or directory
168 ls: a: No such file or directory
169 ls: s: No such file or directory
170 ls: f: No such file or directory
171 ls: d: No such file or directory
172 ls: /store: No such file or directory
173 
174 ls: s: No such file or directory
175 ls: t: No such file or directory
176 ls: o: No such file or directory
177 ls: r: No such file or directory
178 ls: e: No such file or directory
179 ls: a: No such file or directory
180 ls: s: No such file or directory
181 ls: d: No such file or directory
182 ls: f: No such file or directory
183 ls: a: No such file or directory
184 ls: s: No such file or directory
185 ls: f: No such file or directory
186 ls: d: No such file or directory
187 ls: /store: No such file or directory
188 
189 False
190  """
191  _, _, ret = runEOSCommand( path, 'ls')
192  return ret == 0
193 
#also define an alias for backwards compatibility
eostools.isEOSFile
def isEOSFile
Definition: eostools.py:152
def eostools.isFile (   path) 
Returns True if a path is a file.

Tests for file existence.
Returns False for directories.
Works on EOS and local paths.

???This function works with local files, so not the same as isDirectory...
isFile and isDirectory should behave the same.

Definition at line 273 of file eostools.py.

273 
274 def isFile(path):
275  """Returns True if a path is a file.
276 
277  Tests for file existence.
278  Returns False for directories.
279  Works on EOS and local paths.
280 
281  ???This function works with local files, so not the same as isDirectory...
282  isFile and isDirectory should behave the same.
283  """
284 
285  if not path.startswith('/eos') and not path.startswith('/store'):
286  if( os.path.isfile(path) ):
287  return True
288  else:
289  return False
290  else:
291  out, _, _ = runXRDCommand(path,'existfile')
292  return 'The file exists' in out
eostools.isFile
def isFile
Definition: eostools.py:273
eostools.runXRDCommand
def runXRDCommand
Definition: eostools.py:22
reco::if
if(dp >Float(M_PI)) dp-
def eostools.isLFN (   path) 
Tests whether this path is a CMS LFN (name starts with /store...)

Definition at line 56 of file eostools.py.

Referenced by lfnToEOS().

56 
57 def isLFN( path ):
58  """Tests whether this path is a CMS LFN (name starts with /store...)"""
59  # return re.match('^/store.*', path ) is not None
60  return path.startswith('/store')
eostools.isLFN
def isLFN
Definition: eostools.py:56
def eostools.lfnToEOS (   path) 
Converts LFN to EOS.

If path is not an LFN in the first place, return path.
??? shouldn't we raise an exception?

Definition at line 107 of file eostools.py.

References isLFN().

Referenced by cat().

107 
108 def lfnToEOS( path ):
109  """Converts LFN to EOS.
110 
111  If path is not an LFN in the first place, return path.
112  ??? shouldn't we raise an exception?"""
113  if isLFN(path):
114  pfn = 'root://eoscms.cern.ch//eos/cms/' + path
115  return pfn.replace('//store','/store')
116  else:
117  return path
118 
#also define an alias for backwards compatibility
eostools.lfnToEOS
def lfnToEOS
Definition: eostools.py:107
eostools.isLFN
def isLFN
Definition: eostools.py:56
def eostools.lfnToPFN (   path,
  tfcProt = 'rfio' 
) 
Converts an LFN to a PFN. For example:
/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21
->
root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms

This function only checks path, and does not access the storage system.
If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used.
Otherwise, is uses the default one. 

??? what is tfcprot?

Definition at line 76 of file eostools.py.

Referenced by isEOSDir(), runEOSCommand(), and runXRDCommand().

76 
77 def lfnToPFN( path, tfcProt = 'rfio'):
78  """Converts an LFN to a PFN. For example:
79  /store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21
80  ->
81  root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms
82 
83  This function only checks path, and does not access the storage system.
84  If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used.
85  Otherwise, is uses the default one.
86 
87  ??? what is tfcprot? """
88 
89  if path.startswith("/store/"):
90  path = path.replace("/store/","root://eoscms.cern.ch//eos/cms/store/")
91  if path.startswith("/pnfs/psi.ch/cms/trivcat/"):
92  path = path.replace("/pnfs/psi.ch/cms/trivcat/","root://t3se01.psi.ch//")
93  #print "path to cmsFile():", path
94  entity = cmsIO.cmsFile( path, tfcProt )
95 # tokens = cmsIO.splitPFN(entity.pfn)
96  pfn = '%s://%s//%s/' % (entity.protocol,entity.host,entity.path)
97 
98  pfn = entity.pfn
99  if tfcProt == 'rfio' and \
100  entity.path.startswith("/eos/cms/") and \
101  str(entity.stat()).startswith("Error 3011: Unable to stat"):
102 
103  pfn.replace("/eos/cms","/castor/cern.ch/cms")
104  pfn.replace("eoscms","castorcms")
105  return pfn
106 
eostools.lfnToPFN
def lfnToPFN
Definition: eostools.py:76
def eostools.listFiles (   path,
  rec = False,
  full_info = False 
) 
Provides a list of the specified directory

Definition at line 299 of file eostools.py.

299 
300 def listFiles(path, rec = False, full_info = False):
301  """Provides a list of the specified directory
302  """
303  # -- listing on the local filesystem --
304  if os.path.isdir( path ):
305  if not rec:
306  # not recursive
307  return [ '/'.join([path,file]) for file in os.listdir( path )]
308  else:
309  # recursive, directories are put in the list first,
310  # followed by the list of all files in the directory tree
311  result = []
312  allFiles = []
313  for root,dirs,files in os.walk(path):
314  result.extend( [ '/'.join([root,dir]) for dir in dirs] )
315  allFiles.extend( [ '/'.join([root,file]) for file in files] )
316  result.extend(allFiles)
317  return result
318  # -- listing on EOS --
319  cmd = 'dirlist'
320  if rec:
321  cmd = 'dirlistrec'
322  files, _, _ = runXRDCommand(path, cmd)
323  result = []
324  for line in files.split('\n'):
325  tokens = [t for t in line.split() if t]
326  if tokens:
327  #convert to an LFN
328  # result.append(tuple(tokens))
329  #COLIN need same interface for eos and local fs
330  if full_info:
331  result.append( tokens)
332  else:
333  result.append( tokens[4] )
334  return result
eostools.listFiles
def listFiles
Definition: eostools.py:299
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
eostools.runXRDCommand
def runXRDCommand
Definition: eostools.py:22
def eostools.ls (   path,
  rec = False 
) 
Provides a simple list of the specified directory, works on EOS and locally

Definition at line 348 of file eostools.py.

Referenced by FWPFCandidateDetailView.addClusters(), HcalLaserEventFilter2012.addEventString(), LumiMonitor.analyze(), PSMonitor.analyze(), DisplayGeom.analyze(), TriggerBxVsOrbitMonitor.analyze(), PixelLumiDQM.beginLuminosityBlock(), DQMProvInfo.blankPreviousLumiSections(), FWBeamSpotProxyBuilder.build(), FWPFClusterRPZProxyBuilder.build(), FWSiPixelClusterProxyBuilder.build(), FWPFClusterRPZUtils.buildRhoPhiClusterLineSet(), FWPFClusterRPZUtils.buildRhoZClusterLineSet(), evf::EvFDaqDirector.bumpFile(), L1TSync.certifyLSBlock(), CompositeLogicalTrajectoryFilter.CompositeLogicalTrajectoryFilter(), L1TSync.doFractionInSync(), PixelLumiDQM.endLuminosityBlock(), L1TRate.endLuminosityBlock(), RawEventFileWriterForBU.endOfLS(), RateBuffer.fill(), EcalDeadCellTriggerPrimitiveFilter.filter(), RawEventFileWriterForBU.finishFileWrite(), RateBuffer.getLSAlgoRate(), RateBuffer.getLSRate(), RateBuffer.getLSTechRate(), BeamSpotWorkflow.getNewRunList(), IdealZDCTrapezoid.initCorners(), IdealCastorTrapezoid.initCorners(), EcalDeadCellTriggerPrimitiveFilter.loadEventInfo(), FWBeamSpotProxyBuilder.localModelChanges(), ls_cert_type(), lsbs_cert(), copyAndRename.main(), copyFromCastor.main(), rename.main(), copyFiles.main(), main(), splitter.main(), checkRuns.main(), nlumis(), evf::EvFDaqDirector.preGlobalEndLumi(), AlpgenSource.readAlpgenEvent(), FedRawDataInputSource.readSupervisor(), evf::FastMonitoringService.reportLockWait(), lumi::TRGScalers2DB.retrieveData(), GenericTripletGenerator.run(), GenericPairGenerator.run(), BeamHaloPairGenerator.run(), fffnaming.runLumiPrefix(), fffnaming.runLumiPrefixFill(), MonitorElement.setLumi(), DIPLumiDetail.setOrigin(), DIPLumiSummary.setOrigin(), FWPFClusterRPZProxyBuilder.sharedBuild(), TT6CMNSubtractor.subtract_(), IteratedMedianCMNSubtractor.subtract_(), SimpleCosmicBONSeeder.triplets(), evf::EvFDaqDirector.updateFuLock(), DTTimeEvolutionHisto.updateTimeSlot(), and RawEventOutputModuleForBU< Consumer >.write().

348 
349 def ls(path, rec = False):
350  """Provides a simple list of the specified directory, works on EOS and locally"""
351  return [eosToLFN(t) for t in listFiles(path, rec)]
eostools.ls
def ls
Definition: eostools.py:348
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.listFiles
def listFiles
Definition: eostools.py:299
def eostools.ls_EOS (   path,
  rec = False 
) 
Provides a simple list of the specified directory, works on EOS only, but is faster than the xrd version

Definition at line 352 of file eostools.py.

Referenced by datasetNotEmpty().

352 
353 def ls_EOS(path, rec = False):
354  """Provides a simple list of the specified directory, works on EOS only, but is faster than the xrd version"""
355  if rec:
356  stdout, _, ret = runEOSCommand(path,'find','-f')
357  return [eosToLFN(line) for line in stdout.split('\n') if line]
358  else:
359  stdout, _, ret = runEOSCommand(path,'ls')
360  lfn = eosToLFN(path)
361  return [os.path.join(lfn,line) for line in stdout.split('\n') if line]
eostools.ls_EOS
def ls_EOS
Definition: eostools.py:352
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.runEOSCommand
def runEOSCommand
Definition: eostools.py:38
def eostools.matchingFiles (   path,
  regexp 
) 
Return a list of files matching a regexp

Definition at line 518 of file eostools.py.

518 
519 def matchingFiles( path, regexp):
520  """Return a list of files matching a regexp"""
521 
522  # print path, regexp
523  pattern = re.compile( regexp )
524  #files = ls_EOS(path)
525  files = ls(path)
526  # print files
527  return [f for f in files if pattern.match(os.path.basename(f)) is not None]
eostools.ls
def ls
Definition: eostools.py:348
eostools.matchingFiles
def matchingFiles
Definition: eostools.py:518
def eostools.mkdir (   path) 
Create a directory, either on EOS or locally

Definition at line 250 of file eostools.py.

Referenced by L1GtVhdlWriter.analyze(), spu.create_dir(), dd_to_html(), MillePedeMonitor.init(), evf::EvFDaqDirector.initRun(), LaserSorter.LaserSorter(), L1GtVhdlWriterCore.makeFirmware(), pos::PixelConfigFile.makeNewVersion(), L1MuGlobalMuonTrigger.produce(), EgammaObjects.saveHistos(), VariablePlotter.setDir(), LaserSorter.streamFileName(), WatcherStreamFileReader.WatcherStreamFileReader(), and RawEventFileWriterForBU.writeJsds().

250 
251 def mkdir(path):
252  """Create a directory, either on EOS or locally"""
253  # print 'mkdir', path
254  if isEOS( path ) or isLFN(path):
255  createEOSDir(path)
256  else:
257  # recursive directory creation (like mkdir -p)
258  os.makedirs(path)
259  return path
eostools.isLFN
def isLFN
Definition: eostools.py:56
eostools.mkdir
def mkdir
Definition: eostools.py:250
eostools.isEOS
def isEOS
Definition: eostools.py:61
eostools.createEOSDir
def createEOSDir
Definition: eostools.py:229
def eostools.move (   src,
  dest 
) 
Move filename1 to filename2 locally to the same server

Definition at line 510 of file eostools.py.

Referenced by CompatibleDetToGroupAdder.add(), tracking::TempMeasurements.add(), TkGluedMeasurementDet::HitCollectorForFastMeasurements.add(), edm::Principal.addAliasedProduct(), TrajectorySegmentBuilder.addGroup(), CosmicTrajectoryBuilder.AddHit(), edm::Principal.addInputProduct(), edmNew::DetSetVector< edm::Ref< edmNew::DetSetVector< SiStripCluster >, SiStripCluster, edmNew::DetSetVector< SiStripCluster >::FindForDetSetVector > >.addItem(), edm::Principal.addProductOrThrow(), TkGluedMeasurementDet::HitCollectorForRecHits.addProjected(), edm::Principal.addScheduledProduct(), edm::Principal.addSourceProduct(), BaseCkfTrajectoryBuilder.addToResult(), edm::Principal.addUnscheduledProduct(), GroupedCkfTrajectoryBuilder.advanceOneLayer(), MultiTrackValidator.analyze(), DTCCBConfig.appendConfigKey(), LHESource.beginRun(), JME::bimap< Binning, std::string >.bimap(), DQMStore.book(), PileupJetIdAlgo.bookReader(), Basic2DGenericPFlowClusterizer.buildClusters(), PFlow2DClusterizerWithTime.buildClusters(), DetIdAssociator.buildMap(), SeedFromConsecutiveHitsCreator.buildSeed(), TrackProducerAlgorithm< reco::Track >.buildTrack(), TrackProducerAlgorithm< reco::GsfTrack >.buildTrack(), DTReadOutMapping.cacheMap(), TrackingParticleNumberOfLayers.calculate(), clangcms::FiniteMathChecker.checkPreStmt(), XrdAdaptor::RequestManager.checkSourcesImpl(), DAClusterizerInZ_vect.clusterize(), PixelThresholdClusterizer.clusterizeDetUnit(), edm::LuminosityBlock.commit_(), edm::Run.commit_(), edm::Event.commit_aux(), CaloTowersCreationAlgo.convert(), edm.convert_handle(), edm::SharedResourcesRegistry.createAcquirer(), edm::SharedResourcesRegistry.createAcquirerForSourceDelayedReader(), edm::impl.createCommunicatorIfNeeded(), StandAloneMuonFilter.createDefaultTrajectory(), edm::OutputModuleCommunicatorT< T >.createIfNeeded(), BareRootProductGetter.createNewBuffer(), ESDigitizer.createNoisyList(), edm::maker::ModuleHolderT< T >.createOutputModuleCommunicator(), CosmicTrajectoryBuilder.createStartingTrajectory(), CRackTrajectoryBuilder.createStartingTrajectory(), KinematicTree.currentProductionVertex(), PFRecoTauDiscriminationByIsolation.discriminate(), edm::RecordInParentless< PROD >.do_it(), edm::RecordInParentfull< PROD >.do_it(), OutsideInMuonSeeder.doLayer(), EcalTBDigiProducer.EcalTBDigiProducer(), MillePedeFileConverter.endLuminosityBlockProduce(), LHESource.endRun(), MeasurementDet.fastMeasurements(), lhef::LHEReader::FileSource.FileSource(), edm::RootOutputFile.fillBranches(), HcalGeometry.fillDetIds(), edm::RootFile.fillIndexIntoFile(), PhotonMVAEstimatorRun2Phys14NonTrig.fillMVAVariables(), PhotonMVAEstimatorRun2Spring15NonTrig.fillMVAVariables(), ElectronMVAEstimatorRun2Phys14NonTrig.fillMVAVariables(), ElectronMVAEstimatorRun2Spring15Trig.fillMVAVariables(), ElectronMVAEstimatorRun2Spring15NonTrig.fillMVAVariables(), ElectronMVAEstimatorRun2Spring16GeneralPurpose.fillMVAVariables(), ElectronMVAEstimatorRun2Spring16HZZ.fillMVAVariables(), HPDNoiseGenerator.fillNoiseSignals(), BadGlobalMuonTagger.filter(), L1GTPrescaler.filter(), L1TGlobalPrescaler.filter(), TkStripMeasurementDet.filteredRecHits(), cms::SiPixelDigitizer.finalizeEvent(), LocalFileSystem.findCachePath(), LocalMaximumSeedFinder.findSeeds(), V0Fitter.fitAll(), edm::ProvenanceAdaptor.fixProcessHistory(), reco::FormulaEvaluator.FormulaEvaluator(), edm::EventBase.get(), edm::eventsetup::EventSetupRecord.get(), edm::Event.get(), edm.getAnyPtr(), PileUpEventPrincipal.getByLabel(), edm::EventBase.getByLabel(), edm::RunBase.getByLabel(), edm::LuminosityBlockBase.getByLabel(), edm::LuminosityBlock.getByLabel(), edm::Run.getByLabel(), edm::Event.getByLabel(), edm::LuminosityBlock.getByToken(), edm::Run.getByToken(), edm::Event.getByToken(), edm::PrincipalGetAdapter.getManyByType(), SeedFinder.getSeed(), edm::StreamerOutputModuleBase.getTriggerResults(), edm::OutputModule.getTriggerResults(), edm.getWrapperBasePtr(), PixelForwardLayer.groupedCompatibleDetsV(), PixelForwardLayerPhase1.groupedCompatibleDetsV(), TECLayer.groupedCompatibleDetsV(), CompositeTECWedge.groupedCompatibleDetsV(), Phase2OTEndcapLayer.groupedCompatibleDetsV(), TIDRing.groupedCompatibleDetsV(), TBLayer.groupedCompatibleDetsV(), TIBRing.groupedCompatibleDetsV(), Phase2OTEndcapRing.groupedCompatibleDetsV(), PixelBlade.groupedCompatibleDetsV(), TOBRod.groupedCompatibleDetsV(), Phase1PixelBlade.groupedCompatibleDetsV(), Phase2OTBarrelRod.groupedCompatibleDetsV(), CompositeTECPetal.groupedCompatibleDetsV(), LayerMeasurements.groupedMeasurements(), XrdAdaptor::RequestManager.handle(), XrdAdaptor::RequestManager::OpenHandler.HandleResponseWithHosts(), SeedingLayerSetsBuilder.hits(), RectangularEtaPhiTrackingRegion.hits(), MultiHitGeneratorFromChi2.hitSets(), HLTConfigDataEx.HLTConfigDataEx(), PFHBHERecHitCreator.importRecHits(), PFHcalRecHitCreator< Digi, Geometry, Layer, Detector >.importRecHits(), PFEcalRecHitCreator< Geometry, Layer, Detector >.importRecHits(), PFPSRecHitCreator.importRecHits(), PFHFRecHitCreator.importRecHits(), HitTripletGeneratorFromPairAndLayers.init(), MultiHitGeneratorFromPairAndLayers.init(), edm::EventProcessor.init(), DTCCBConfig.initialize(), XrdAdaptor::RequestManager.initialize(), initializeDummyPSet(), PFEGammaAlgo.initializeProtoCands(), TransientInitialStateEstimator.innerState(), edm::ParentageRegistry.insertMapped(), GeomDetCompatibilityChecker.isCompatible(), JME::JetParameters.JetParameters(), L1MuBMTFSetup.L1MuBMTFSetup(), L1MuDTTFSetup.L1MuDTTFSetup(), SeedingLayerSetsBuilder::LayerSpec.LayerSpec(), CkfTrajectoryBuilder.limitedCandidates(), edm.makeESHandleExceptionFactory(), edm.makeHandleExceptionFactory(), edm::RootFile.makeProvenanceReaderMaker(), TSGForOI.makeSeedsFromHits(), TrajectoryFitter.makeVect(), TkPixelMeasurementDet.measurements(), DQMStore.mergeAndResetMEsLuminositySummaryCache(), DQMStore.mergeAndResetMEsRunSummaryCache(), edm::ProductHolderBase.mergeProduct(), edm::InputProductHolder.mergeProduct_(), edm::ProducedProductHolder.mergeProduct_(), edm::AliasProductHolder.mergeProduct_(), DetGroupMerger.mergeTwoLevels(), KinematicTree.movePointerToTheTop(), edm::soahelper::SoATupleHelper< I, Args >.moveToNew(), BaseCkfTrajectoryBuilder.moveToResult(), MuonAssociatorByHits.MuonAssociatorByHits(), MuonToSimAssociatorByHits.MuonToSimAssociatorByHits(), PhotonMVAEstimatorRun2Phys14NonTrig.mvaValue(), PhotonMVAEstimatorRun2Spring15NonTrig.mvaValue(), ElectronMVAEstimatorRun2Phys14NonTrig.mvaValue(), ElectronMVAEstimatorRun2Spring15Trig.mvaValue(), ElectronMVAEstimatorRun2Spring15NonTrig.mvaValue(), ElectronMVAEstimatorRun2Spring16GeneralPurpose.mvaValue(), ElectronMVAEstimatorRun2Spring16HZZ.mvaValue(), LocalStorageMaker.open(), DCacheStorageMaker.open(), XrdStorageMaker.open(), StormStorageMaker.open(), StormLcgGtStorageMaker.open(), StorageFactory.open(), RFIOStorageMaker.open(), CaloTDigitizerDefaultRun< Traits >.operator()(), extstd::clone_ptr< Bounds >.operator=(), DetGroupElement.operator=(), edm::InputTag.operator=(), ProxyBase11< T >.operator=(), reco::TransientTrack.operator=(), edm::RefVectorBase< key_type >.operator=(), edm::RefVector< JetEisolAssociationCollection >.operator=(), pat::PackedGenParticle.operator=(), edmNew::dstvdetails::DetSetVectorTrans::Item.operator=(), DetGroupMerger.orderAndMergeTwoLevels(), PFRecoTauDiscriminationByIsolation.PFRecoTauDiscriminationByIsolation(), PixelTrackReconstruction.PixelTrackReconstruction(), TFWLiteSelectorBasic.Process(), edm::SubProcess.process(), MultiTrackSelector.processMVA(), ShallowEventDataProducer.produce(), QualityFilter.produce(), TrackingParticleConversionSelector.produce(), edm::EventAuxiliaryHistoryProducer.produce(), SeedCombiner.produce(), TrackingParticleNumberOfLayersProducer.produce(), JetCollectionReducerT< T >.produce(), GeneratorSmearedProducer.produce(), PFMatchedCandidateRefExtractor.produce(), ParticleLevelProducer.produce(), EmptySimHits.produce(), evf::EvFFEDSelector.produce(), VertexAssociatorByTracksProducer.produce(), VertexAssociatorByPositionAndTracksProducer.produce(), PFMuonUntagger.produce(), CandIsoDepositProducer.produce(), MuIsoDepositCopyProducer.produce(), pat::PackedPFCandidateRefMixer.produce(), FastTrackerRecHitCombiner.produce(), JetTracksAssociationToTrackRefs.produce(), PFCandidateMuonUntagger.produce(), FastTrackerRecHitMaskProducer.produce(), BaseEvtVtxGenerator.produce(), HiFJGridEmptyAreaCalculator.produce(), HcalRawToDigi.produce(), HiFJRhoProducer.produce(), MuonReSeeder.produce(), CTPPSDiamondLocalTrackFitter.produce(), pat::L1MuonMatcher.produce(), GsfElectronCoreGSCrysFixer.produce(), TrackListMerger.produce(), CombinedRecHitCollectionProducer.produce(), ClusterTPAssociationProducer.produce(), CTPPSDiamondRecHitProducer.produce(), CorrMETDataExtractor.produce(), FastTrackerRecHitMatcher.produce(), MixEvtVtxGenerator.produce(), edm::LogErrorHarvester.produce(), JetCorrectorProducer< T >.produce(), PhotonGSCrysSimpleFixer.produce(), edm::TriggerResultInserter.produce(), GsfEleGSCrysSimpleFixer.produce(), FakeTrackProducer< T >.produce(), TrackMVAClassifierBase.produce(), GsfElectronGSCrysFixer.produce(), TrackCandidateProducer.produce(), TrackMCQuality.produce(), TrackAssociatorByPositionProducer.produce(), HLTScoutingMuonProducer.produce(), QuickTrackAssociatorByHitsProducer.produce(), EGRefinedSCFixer.produce(), EGGainSwitchFlagProducer< C >.produce(), ExtraFromSeeds.produce(), DeepNNTagInfoProducer.produce(), HLTScoutingEgammaProducer.produce(), TrackAssociatorByChi2Producer.produce(), GenHIEventProducer.produce(), MixBoostEvtVtxGenerator.produce(), TrackAssociatorByHitsProducer.produce(), SiTrackerGaussianSmearingRecHitConverter.produce(), TracksToTrajectories.produce(), BetaBoostEvtVtxGenerator.produce(), CorrectedECALPFClusterProducer.produce(), PixelJetPuId.produce(), PFEGammaProducer.produce(), DeepCMVATagInfoProducer.produce(), BoostedDoubleSVProducer.produce(), DeepFlavourJetTagsProducer.produce(), MuonIdProducer.produce(), TrajectorySeedProducer.produce(), ChainedJetCorrectorProducer.produce(), SmearedJetProducerT< T >.produce(), MuonToTrackingParticleAssociatorEDProducer.produce(), cms::CkfTrackCandidateMakerBase.produceBase(), edm::ProductProvenance.ProductProvenance(), TkGluedMeasurementDet.projectOnGluedDet(), reco::PFCluster.pruneUsing(), TempTrajectory.push(), edmNew::DetSetVector< T >::FastFiller.push_back(), edmNew::DetSetVector< T >::TSFastFiller.push_back(), edm::RunPrincipal.put(), edm::LuminosityBlockPrincipal.put(), edm::Run.put(), edm::LuminosityBlock.put(), edm::Event.put(), edm::EventPrincipal.put(), KfTrackProducerBase.putInEvt(), GsfTrackProducerBase.putInEvt(), DAFTrackProducer.putInEvtTrajAnn(), edm::EventPrincipal.putOnRead(), edm::Principal.putOrMerge(), edm::ProductHolderBase.putProduct(), edm::InputProductHolder.putProduct_(), edm::ProducedProductHolder.putProduct_(), edm::AliasProductHolder.putProduct_(), edm::DataMixingTrackingParticleWorker.putTrackingParticle(), FedRawDataInputSource.read(), edm::StreamerInputSource.read(), edm::RootFile.readCurrentEvent(), LHESource.readEvent_(), edm::PoolSource.readFile_(), edm::EventPrincipal.readFromSource_(), GroupedCkfTrajectoryBuilder.rebuildSeedingRegion(), GroupedCkfTrajectoryBuilder.rebuildTrajectories(), SimpleDAFHitCollector.recHits(), TkStripMeasurementDet.recHits(), RecoTauDiscriminantCutMultiplexer.RecoTauDiscriminantCutMultiplexer(), TrajectorySegmentBuilder.redoMeasurements(), edm::ESProxyFactoryProducer.registerFactoryWithKey(), VertexFitterManager.registerFitter(), edm::Principal.resolveProductImmediately(), ESDigitizer.run(), PFEGammaToCandidateRemapper.run(), MultiTrackSelector.run(), PixelForwardLayer.searchNeighbors(), PixelForwardLayerPhase1.searchNeighbors(), TIBRing.searchNeighbors(), CRackTrajectoryBuilder.seedMeasurements(), CaloTower.setConstituents(), edm::storage::StatisticsSenderService.setCurrentServer(), reco::PFTau.setisolationPiZeroCandidates(), reco::PFTau.setIsolationPiZeroCandidatesRefs(), reco::PFTau.setIsolationTauChargedHadronCandidates(), reco::PFTau.setIsolationTauChargedHadronCandidatesRefs(), reco::formula::BinaryOperatorEvaluatorBase.setLeftEvaluator(), edm::InputProductHolder.setProduct(), reco::formula::BinaryOperatorEvaluatorBase.setRightEvaluator(), reco::PFTau.setsignalPiZeroCandidates(), reco::PFTau.setSignalPiZeroCandidatesRefs(), reco::PFTau.setSignalTauChargedHadronCandidates(), reco::PFTau.setSignalTauChargedHadronCandidatesRefs(), L1MuDTTrackFinder.setup(), HLTJets.setup(), L1MuBMTrackFinder.setup(), edm::ProductData.setWrapper(), edm::service::SiteLocalConfigService.SiteLocalConfigService(), pat::L1MuonMatcher.storeExtraInfo(), DDValue::StringHolder.StringHolder(), edm::SubProcess.SubProcess(), TempTrajectory.TempTrajectory(), CSCSegtoRPC.thePoints(), DTSegtoRPC.thePoints(), TracktoRPC.thePoints(), TrackingLayer.toIdString(), TrackingLayer.toString(), edm::service::Tracer.Tracer(), TrajectorySmoother.trajectories(), TrajectoryBuilder.trajectories(), edm::ProductData.unsafe_setWrapper(), MuonTrajectoryUpdator.update(), TrajectorySegmentBuilder.updateCandidatesWithBestHit(), TrajectorySegmentBuilder.updateTrajectory(), CkfTrajectoryBuilder.updateTrajectory(), edm::detail::TriggerResultsBasedEventSelector.wantEvent(), StorageFactory.wrapNonLocalFile(), PFMuonUntagger.writeAssociation(), VirtualJetProducer.writeJets(), PFCandidateMuonUntagger.writeValueMap(), PFMuonUntagger.writeValueMap(), l1t::stage2::GMTCollections.~GMTCollections(), TrackCollectionCloner::Producer.~Producer(), and edmNew::DetSetVector< T >::TSFastFiller.~TSFastFiller().

510 
511 def move(src, dest):
512  """Move filename1 to filename2 locally to the same server"""
513 
514  src = eosToLFN(src)
515  dest = eosToLFN(dest)
516 
517  runXRDCommand(src,'mv', lfnToEOS(dest))
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.lfnToEOS
def lfnToEOS
Definition: eostools.py:107
eostools.move
def move
Definition: eostools.py:510
eostools.runXRDCommand
def runXRDCommand
Definition: eostools.py:22
def eostools.remove (   files,
  rec = False 
) 
Remove a list of files and directories, possibly recursively

Colin: Is that obsolete? why not use rm?

Definition at line 381 of file eostools.py.

381 
382 def remove( files, rec = False):
383  """Remove a list of files and directories, possibly recursively
384 
385  Colin: Is that obsolete? why not use rm?"""
386  for path in files:
387  lfn = eosToLFN(path)
388  if not rec:
389  rm(path)
390  else:
391  #this should be used with care
392  file_list = ls(path, rec = True)
393  file_list.append(lfn)
394 
395  #order the files in depth order - i.e. remove the deepest files first
396  files_rec = sorted([(len([ff for ff in f.split('/') if ff]), f) for f in file_list if f and f.startswith(lfn)], reverse = True)
397 
398  for f in files_rec:
399  rm(f[1])
eostools.ls
def ls
Definition: eostools.py:348
eostools.remove
def remove
Definition: eostools.py:381
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.rm
def rm
Definition: eostools.py:362
def eostools.rm (   path,
  rec = False 
) 
rm, works on EOS and locally.

Colin: should implement a -f mode and a confirmation when deleting dirs recursively.

Definition at line 362 of file eostools.py.

Referenced by remove().

362 
363 def rm(path, rec=False):
364  """rm, works on EOS and locally.
365 
366  Colin: should implement a -f mode and a confirmation when deleting dirs recursively."""
367  # print 'rm ', path
368  path = lfnToEOS(path)
369  if isEOS(path):
370  if rec:
371  runEOSCommand(path, 'rm', '-r')
372  else:
373  runEOSCommand(path,'rm')
374  elif os.path.exists(path):
375  if not rec:
376  os.remove( path )
377  else:
378  shutil.rmtree(path)
379  else:
380  raise ValueError(path + ' is not EOS and not local... should not happen!')
eostools.lfnToEOS
def lfnToEOS
Definition: eostools.py:107
eostools.runEOSCommand
def runEOSCommand
Definition: eostools.py:38
eostools.rm
def rm
Definition: eostools.py:362
eostools.isEOS
def isEOS
Definition: eostools.py:61
def eostools.runEOSCommand (   path,
  cmd,
  args 
) 
Run an eos command.

!!! Will, when the EOS command fails, it passes silently...
I think we should really try and raise an exception in case of problems.
should be possible as the return code is provided in the tuple returned by runner.

Definition at line 38 of file eostools.py.

References eosToLFN(), and lfnToPFN().

Referenced by chmod(), createEOSDir(), eosDirSize(), ls_EOS(), and rm().

38 
39 def runEOSCommand(path, cmd, *args):
40  """Run an eos command.
41 
42  !!! Will, when the EOS command fails, it passes silently...
43  I think we should really try and raise an exception in case of problems.
44  should be possible as the return code is provided in the tuple returned by runner."""
45 
46  lfn = eosToLFN(path)
47  pfn = lfnToPFN(lfn)
48  tokens = cmsIO.splitPFN(pfn)
49 
50  #obviously, this is not nice
51  command = [eos_select, cmd]
52  command.extend(args)
53  command.append(tokens[2])
54  runner = cmsIO.cmsFileManip()
55  return runner.runCommand(command)
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.runEOSCommand
def runEOSCommand
Definition: eostools.py:38
eostools.lfnToPFN
def lfnToPFN
Definition: eostools.py:76
def eostools.runXRDCommand (   path,
  cmd,
  args 
) 
Run an xrd command.

!!! Will, what is happening in case of problem?
??? At some point, should return a list of lines instead of a string.

Definition at line 22 of file eostools.py.

References eosToLFN(), and lfnToPFN().

Referenced by cat(), isDirectory(), isFile(), listFiles(), and move().

22 
23 def runXRDCommand(path, cmd, *args):
24  """Run an xrd command.
25 
26  !!! Will, what is happening in case of problem?
27  ??? At some point, should return a list of lines instead of a string."""
28 
29  lfn = eosToLFN(path)
30  #print "lfn:", lfn, cmd
31  tokens = cmsIO.splitPFN(lfnToPFN(lfn))
32 
33  command = ['xrd', tokens[1], cmd, tokens[2]]
34  command.extend(args)
35  runner = cmsIO.cmsFileManip()
36  # print ' '.join(command)
37  return runner.runCommand(command)
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.runXRDCommand
def runXRDCommand
Definition: eostools.py:22
eostools.lfnToPFN
def lfnToPFN
Definition: eostools.py:76
def eostools.setCAFPath ( ) 
Hack to get the CAF scripts on the PYTHONPATH

Definition at line 13 of file eostools.py.

13 
14 def setCAFPath():
15  """Hack to get the CAF scripts on the PYTHONPATH"""
16  caf = '/afs/cern.ch/cms/caf/python'
17 
18  if caf not in sys.path:
19  sys.path.append(caf)
setCAFPath()
eostools.setCAFPath
def setCAFPath
Definition: eostools.py:13
def eostools.which (   cmd)

Definition at line 335 of file eostools.py.

Referenced by OOTPileupCorrData.apply(), MuonResidualsFitter.computeHistogramRangeAndBinning(), gen::EvtGenInterface.decay(), pat::PATSingleVertexSelector.filter_(), ConstantStepOdeSolver.getCoordinate(), OOTPileupCorrData.getCorrectionByID(), HBHENegativeEFilter.getEtaIndex(), LMFDat.getNeighbour(), AbsElectronicODERHS.getParameter(), ConstantStepOdeSolver.getPeakTime(), ConstantStepOdeSolver.integrateCoordinate(), ConstantStepOdeSolver.interpolateCoordinate(), MuonResidualsTwoBin.mean(), PiecewiseScalingPolynomial.operator()(), MuonResidualsTwoBin.read(), InputTagDistributorService.retrieve(), TrackMultiSelector.select(), MuonResidualsFitter.selectPeakResiduals(), AbsElectronicODERHS.setParameter(), MuonResidualsTwoBin.stdev(), ConstantStepOdeSolver.truncateCoordinate(), MuonResidualsTwoBin.wmean(), MuonResidualsTwoBin.write(), MuonResidualsFitter.write(), and ConstantStepOdeSolver.writeHistory().

335 
336 def which(cmd):
337  command = ['which', cmd]
338  runner = cmsIO.cmsFileManip()
339  out, _, _ = runner.runCommand(command)
340 
341  lines = [line for line in out.split('\n') if line]
342  if len(lines) == 1:
343  return lines[0]
344  elif len(lines) == 2:
345  return lines[1]
346  else:
347  return lines
eostools.which
def which
Definition: eostools.py:335
def eostools.xrdcp (   src,
  dest 
) 
Does a copy of files using xrd.

Colin: implement a generic cp interface as done for rm, ls, etc?

Definition at line 429 of file eostools.py.

429 
430 def xrdcp(src, dest):
431  """Does a copy of files using xrd.
432 
433  Colin: implement a generic cp interface as done for rm, ls, etc?"""
434 
435  recursive = False
436 
437  #first the src file
438  pfn_src = src
439  if os.path.exists(src):
440  #local
441  pfn_src = src
442  if os.path.isdir(src):
443  recursive = True
444  elif fileExists(src):
445  src = eosToLFN(src)
446  pfn_src = lfnToPFN(src)
447  if isDirectory(src):
448  recursive = True
449  else:
450  raise ValueError(src + ' does not exist.')
451 
452  #now the dest
453  pfn_dest = dest
454  if isEOSDir(dest):
455  dest = eosToLFN(dest)
456  pfn_dest = lfnToPFN(dest)
457  if isDirectory(dest):
458  tokens = cmsIO.splitPFN(pfn_dest)
459  pfn_dest = '%s://%s//%s/' % (tokens[0],tokens[1],tokens[2])
460  elif os.path.exists(dest):
461  pfn_dest = dest
462 
463  command = ['xrdcp']
464  if recursive:
465  # print 'recursive'
466  topDir = src.rstrip('/').split('/')[-1]
467  if topDir != '.':
468  dest = '/'.join([dest, topDir])
469  # print 'mkdir ' + dest
470  mkdir( dest )
471  files = listFiles(src, rec=True)
472  # pprint.pprint( [file[4] for file in files] )
473  for srcFile in files:
474  # srcFile = file[4]
475  pfnSrcFile = srcFile
476  if isEOSDir(srcFile):
477  srcFile = eosToLFN(srcFile)
478  pfnSrcFile = lfnToPFN(srcFile)
479  destFile = srcFile.replace( src, '' )
480  destFile = '/'.join([dest,destFile])
481  pfnDestFile = destFile
482  if isEOSDir(destFile):
483  lfnDestFile = eosToLFN(destFile)
484  pfnDestFile = lfnToPFN(lfnDestFile)
485  # print 'srcFile', pfnSrcFile
486  # print 'destFile', pfnDestFile
487  if isFile(srcFile):
488  _xrdcpSingleFile( pfnSrcFile, pfnDestFile )
489  else:
490  mkdir(destFile)
491  else:
492  _xrdcpSingleFile( pfn_src, pfn_dest )
493 
eostools.isDirectory
def isDirectory
Definition: eostools.py:260
eostools.fileExists
def fileExists
Definition: eostools.py:197
eostools.eosToLFN
def eosToLFN
Definition: eostools.py:65
eostools.listFiles
def listFiles
Definition: eostools.py:299
eostools.isEOSDir
def isEOSDir
Definition: eostools.py:121
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
eostools.isFile
def isFile
Definition: eostools.py:273
eostools.xrdcp
def xrdcp
Definition: eostools.py:429
eostools._xrdcpSingleFile
def _xrdcpSingleFile
Definition: eostools.py:494
eostools.mkdir
def mkdir
Definition: eostools.py:250
split
double split
Definition: MVATrainer.cc:139
eostools.lfnToPFN
def lfnToPFN
Definition: eostools.py:76

Variable Documentation

eostools.castorToLFN = eosToLFN

Definition at line 74 of file eostools.py.

eostools.createCastorDir = createEOSDir

Definition at line 248 of file eostools.py.

string eostools.eos_select = '/afs/cern.ch/project/eos/installation/cms/bin/eos.select'

Definition at line 11 of file eostools.py.

eostools.isCastorDir = isEOSDir

Definition at line 149 of file eostools.py.

eostools.isCastorFile = isEOSFile

Definition at line 194 of file eostools.py.

eostools.lfnToCastor = lfnToEOS

Definition at line 119 of file eostools.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5