CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
eostools Namespace Reference

Functions

def _xrdcpSingleFile
 
def cat
 
def chmod
 
def cmsStage
 
def createEOSDir
 
def datasetNotEmpty
 
def eosDirSize
 
def eosToLFN
 
def fileExists
 
def isDirectory
 
def isEOS
 
def isEOSDir
 
def isEOSFile
 
def isFile
 
def isLFN
 
def lfnToEOS
 
def lfnToPFN
 
def listFiles
 
def ls
 
def ls_EOS
 
def matchingFiles
 
def mkdir
 
def move
 
def remove
 
def rm
 
def runEOSCommand
 
def runXRDCommand
 
def setCAFPath
 
def which
 
def xrdcp
 

Variables

 castorToLFN = eosToLFN
 
 createCastorDir = createEOSDir
 
 isCastorDir = isEOSDir
 
 isCastorFile = isEOSFile
 
 lfnToCastor = lfnToEOS
 

Function Documentation

def eostools._xrdcpSingleFile (   pfn_src,
  pfn_dest 
)
private
Copies a single file using xrd.

Definition at line 492 of file eostools.py.

493 def _xrdcpSingleFile( pfn_src, pfn_dest):
494  """Copies a single file using xrd."""
495 
496  command = ['xrdcp']
497  command.append(pfn_src)
498  command.append(pfn_dest)
499  # print ' '.join(command)
500  run = True
501  if run:
502  runner = cmsIO.cmsFileManip()
503  out, err, ret = runner.runCommand(command)
504  if err:
505  print >> sys.stderr, out
506  print >> sys.stderr, err
507  return ret
def _xrdcpSingleFile
Definition: eostools.py:492
def eostools.cat (   path)
cat, works on EOS and locally

Definition at line 398 of file eostools.py.

Referenced by MVAJetTagPlotter.analyzeTag(), CSCStripAmpResponse.calculateAmpResponse(), CutBasedElectronID.cicSelection(), CutBasedElectronID.classify(), fwlite::Scanner< Collection >.fillDataSet(), edm::service::ELstatistics.formSummary(), GsfEleMVACut.operator()(), PhoMVACut.operator()(), DDStreamer.parts_read(), DDLLogicalPart.processElement(), AlgorithmCalibration< T, CO >.readCategories(), edm::service::ELstatistics.summaryForJobReport(), and TagProbeFitTreeAnalyzer.TagProbeFitTreeAnalyzer().

399 def cat(path):
400  """cat, works on EOS and locally"""
401  path = lfnToEOS(path)
402  if isEOS(path):
403  #print "the file to cat is:", path
404  out, err, _ = runXRDCommand(path,'cat')
405  lines = []
406  if out:
407  pattern = re.compile('cat returned [0-9]+')
408  for line in out.split('\n'):
409  match = pattern.search(line)
410  if line and match is not None:
411  lines.append(line.replace(match.group(0),''))
412  break
413  else:
414  lines.append(line)
415  if err:
416  print >> sys.stderr, out
417  print >> sys.stderr, err
418  allLines = '\n'.join(lines)
419  if allLines and not allLines.endswith('\n'):
420  allLines += '\n'
421  return allLines
422  else:
423  content = file(path).read()
424  if content and not content.endswith('\n'):
425  content += '\n'
426  return content
def cat
Definition: eostools.py:398
def lfnToEOS
Definition: eostools.py:105
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def runXRDCommand
Definition: eostools.py:20
def isEOS
Definition: eostools.py:59
def eostools.chmod (   path,
  mode 
)
Does chmod on a file or directory

Definition at line 291 of file eostools.py.

Referenced by evf::EvFDaqDirector.EvFDaqDirector(), L1GtVhdlWriterCore.makeFirmware(), evf::EvFDaqDirector.openFULockfileStream(), L1GtVhdlWriterCore.writeAlgoSetup(), L1GtVhdlWriterCore.writeCond2intMap2File(), L1GtVhdlWriterCore.writeCondChipPkg(), L1GtVhdlWriterCore.writeConditionChipSetup(), L1GtVhdlWriterCore.writeDefValPkg(), L1GtVhdlWriterCore.writeEtmSetup(), L1GtVhdlWriterCore.writeMuonSetupVhdl(), and L1GtVhdlWriterCore.writeQsfSetupFiles().

292 def chmod(path, mode):
293  """Does chmod on a file or directory"""
294  #
295  return runEOSCommand(path, 'chmod', '-r', str(mode))
296 
def chmod
Definition: eostools.py:291
def runEOSCommand
Definition: eostools.py:36
def eostools.cmsStage (   absDestDir,
  files,
  force 
)
Runs cmsStage with LFNs if possible

Definition at line 535 of file eostools.py.

536 def cmsStage( absDestDir, files, force):
537  """Runs cmsStage with LFNs if possible"""
538 
539  destIsEOSDir = isEOSDir(absDestDir)
540  if destIsEOSDir:
541  createEOSDir( absDestDir )
542 
543  for fname in files:
544  command = ['cmsStage']
545  if force:
546  command.append('-f')
547  command.append(eosToLFN(fname))
548  command.append(eosToLFN(absDestDir))
549  print ' '.join(command)
550  runner = cmsIO.cmsFileManip()
551  runner.runCommand(command)
def eosToLFN
Definition: eostools.py:63
def isEOSDir
Definition: eostools.py:119
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def cmsStage
Definition: eostools.py:535
def createEOSDir
Definition: eostools.py:227
def eostools.createEOSDir (   path)
Makes a directory in EOS

???Will, I'm quite worried by the fact that if this path already exists, and is
a file, everything will 'work'. But then we have a file, and not a directory,
while we expect a dir...

Definition at line 227 of file eostools.py.

Referenced by cmsStage().

228 def createEOSDir( path ):
229  """Makes a directory in EOS
230 
231  ???Will, I'm quite worried by the fact that if this path already exists, and is
232  a file, everything will 'work'. But then we have a file, and not a directory,
233  while we expect a dir..."""
234  lfn = eosToLFN(path)
235  if not isEOSFile(lfn):
236  # if not isDirectory(lfn):
237  runEOSCommand(lfn,'mkdir','-p')
238  # entity = cmsIO.cmsFile( lfn,"stageout")
239  # entity.mkdir([])
240  # # print 'created ', path
241  if isDirectory(path):
242  return path
243  else:
244  raise OSError('cannot create directory '+ path)
245 
#also define an alias for backwards compatibility
def isEOSFile
Definition: eostools.py:150
def isDirectory
Definition: eostools.py:258
def eosToLFN
Definition: eostools.py:63
def runEOSCommand
Definition: eostools.py:36
def createEOSDir
Definition: eostools.py:227
def eostools.datasetNotEmpty (   path,
  regexp 
)

Definition at line 526 of file eostools.py.

527 def datasetNotEmpty( path, regexp ):
528  pattern = re.compile( regexp )
529  files = ls_EOS(path)
530 
531  for f in files:
532  if pattern.match( os.path.basename(f) ) is not None:
533  return 1
534  return 0
def ls_EOS
Definition: eostools.py:350
def datasetNotEmpty
Definition: eostools.py:526
def eostools.eosDirSize (   path)
Returns the size of a directory on EOS in GB.

Definition at line 213 of file eostools.py.

214 def eosDirSize(path):
215  '''Returns the size of a directory on EOS in GB.'''
216  lfn = eosToLFN(path)
217  res = runEOSCommand(lfn, 'find', '--size')
218  output = res[0].split('\n')
219  size = 0
220  for file in output:
221  try:
222  size += float(file.split('=')[2])
223  except IndexError:
224  pass
225  return size/1024/1024/1024
226 
def eosDirSize
Definition: eostools.py:213
def eosToLFN
Definition: eostools.py:63
def runEOSCommand
Definition: eostools.py:36
double split
Definition: MVATrainer.cc:139
def eostools.eosToLFN (   path)
Converts a EOS PFN to an LFN.

Just strip out /eos/cms from path.
If this string is not found, return path.
??? Shouldn't we raise an exception instead?

Definition at line 63 of file eostools.py.

References python.rootplot.root2matplotlib.replace().

Referenced by cmsStage(), createEOSDir(), eosDirSize(), isEOSDir(), runEOSCommand(), and runXRDCommand().

63 
64 def eosToLFN( path ):
65  """Converts a EOS PFN to an LFN.
66 
67  Just strip out /eos/cms from path.
68  If this string is not found, return path.
69  ??? Shouldn't we raise an exception instead?"""
70  return path.replace('root://eoscms.cern.ch/', '').replace('/eos/cms','')
71 
#also define an alias for backwards compatibility
def eosToLFN
Definition: eostools.py:63
def eostools.fileExists (   path)
Returns true if path is a file or directory stored locally, or on EOS.

This function checks for the file or directory existence.

Definition at line 195 of file eostools.py.

196 def fileExists( path ):
197  """Returns true if path is a file or directory stored locally, or on EOS.
198 
199  This function checks for the file or directory existence."""
200 
201  eos = isEOSDir(path)
202  result = False
203  if eos:
204  # print 'eos', path
205  result = isEOSFile(path)
206  else:
207  # print 'not eos', path
208  #check locally
209  result = os.path.exists(path)
210  # print result
211  return result
212 
def isEOSFile
Definition: eostools.py:150
def fileExists
Definition: eostools.py:195
def isEOSDir
Definition: eostools.py:119
def eostools.isDirectory (   path)
Returns True if path is a directory on EOS.

Tests for file existence. 
This function returns False for EOS files, and crashes with local paths

???Will, this function also seems to work for paths like:
/eos/cms/...
??? I think that it should work also for local files, see isFile.

Definition at line 258 of file eostools.py.

Referenced by createEOSDir().

259 def isDirectory(path):
260  """Returns True if path is a directory on EOS.
261 
262  Tests for file existence.
263  This function returns False for EOS files, and crashes with local paths
264 
265  ???Will, this function also seems to work for paths like:
266  /eos/cms/...
267  ??? I think that it should work also for local files, see isFile."""
268 
269  out, _, _ = runXRDCommand(path,'existdir')
270  return 'The directory exists' in out
def isDirectory
Definition: eostools.py:258
def runXRDCommand
Definition: eostools.py:20
def eostools.isEOS (   path)
Tests whether this path is a CMS EOS (name starts with /eos...)

Definition at line 59 of file eostools.py.

Referenced by cat().

59 
60 def isEOS( path ):
61  """Tests whether this path is a CMS EOS (name starts with /eos...)"""
62  return path.startswith('/eos') or path.startswith('root://eoscms.cern.ch//eos/cms')
def isEOS
Definition: eostools.py:59
def eostools.isEOSDir (   path)
Returns True if path is either:
/store/...
or
/eos/cms/store/...
or
root://eoscms.cern.ch//eos/cms/

Otherwise, returns False.

WARNING!! This function does not check for path existence,
and returns true also for plain files.
!!! Will, is my summary correct? 

Definition at line 119 of file eostools.py.

References eosToLFN(), and lfnToPFN().

Referenced by cmsStage(), and fileExists().

120 def isEOSDir( path ):
121  """Returns True if path is either:
122  /store/...
123  or
124  /eos/cms/store/...
125  or
126  root://eoscms.cern.ch//eos/cms/
127 
128  Otherwise, returns False.
129 
130  WARNING!! This function does not check for path existence,
131  and returns true also for plain files.
132  !!! Will, is my summary correct?
133  """
134  if os.path.exists( path ):
135  # path does not exist
136  # COLIN: I think this condition could be removed,
137  # as it duplicates the following one.
138  return False
139  if not path.startswith('/eos') and not path.startswith('/store') and not path.startswith('root://eoscms.cern.ch//eos/cms/'):
140  # neither an EOS PFN or a LFN.
141  return False
142  # at this stage, we must have an EOS PFN or an LFN
143  pfn = lfnToPFN(eosToLFN(path))
144  tokens = cmsIO.splitPFN(pfn)
145  return tokens and tokens[1].lower().startswith('eos')
146 
#also define an alias for backwards compatibility
def eosToLFN
Definition: eostools.py:63
def isEOSDir
Definition: eostools.py:119
def lfnToPFN
Definition: eostools.py:74
def eostools.isEOSFile (   path,
  tfcProt = 'rfio' 
)
Returns True if path is a file or directory stored on EOS (checks for path existence)
??? This function does not behave well if passed a non EOS path...
returns lots of error messages like:
>>> eostools.isEOSFile('/store/asdfasfd')
Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2
ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory

ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory

False

Definition at line 150 of file eostools.py.

Referenced by createEOSDir(), and fileExists().

151 def isEOSFile( path, tfcProt = 'rfio'):
152  """Returns True if path is a file or directory stored on EOS (checks for path existence)
153  ??? This function does not behave well if passed a non EOS path...
154  returns lots of error messages like:
155 >>> eostools.isEOSFile('/store/asdfasfd')
156 Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2
157 ls: s: No such file or directory
158 ls: t: No such file or directory
159 ls: o: No such file or directory
160 ls: r: No such file or directory
161 ls: e: No such file or directory
162 ls: a: No such file or directory
163 ls: s: No such file or directory
164 ls: d: No such file or directory
165 ls: f: No such file or directory
166 ls: a: No such file or directory
167 ls: s: No such file or directory
168 ls: f: No such file or directory
169 ls: d: No such file or directory
170 ls: /store: No such file or directory
171 
172 ls: s: No such file or directory
173 ls: t: No such file or directory
174 ls: o: No such file or directory
175 ls: r: No such file or directory
176 ls: e: No such file or directory
177 ls: a: No such file or directory
178 ls: s: No such file or directory
179 ls: d: No such file or directory
180 ls: f: No such file or directory
181 ls: a: No such file or directory
182 ls: s: No such file or directory
183 ls: f: No such file or directory
184 ls: d: No such file or directory
185 ls: /store: No such file or directory
186 
187 False
188  """
189  _, _, ret = runEOSCommand( path, 'ls')
190  return ret == 0
191 
#also define an alias for backwards compatibility
def isEOSFile
Definition: eostools.py:150
def eostools.isFile (   path)
Returns True if a path is a file.

Tests for file existence.
Returns False for directories.
Works on EOS and local paths.

???This function works with local files, so not the same as isDirectory...
isFile and isDirectory should behave the same.

Definition at line 271 of file eostools.py.

272 def isFile(path):
273  """Returns True if a path is a file.
274 
275  Tests for file existence.
276  Returns False for directories.
277  Works on EOS and local paths.
278 
279  ???This function works with local files, so not the same as isDirectory...
280  isFile and isDirectory should behave the same.
281  """
282 
283  if not path.startswith('/eos') and not path.startswith('/store'):
284  if( os.path.isfile(path) ):
285  return True
286  else:
287  return False
288  else:
289  out, _, _ = runXRDCommand(path,'existfile')
290  return 'The file exists' in out
def isFile
Definition: eostools.py:271
def runXRDCommand
Definition: eostools.py:20
if(conf.exists("allCellsPositionCalc"))
def eostools.isLFN (   path)
Tests whether this path is a CMS LFN (name starts with /store...)

Definition at line 54 of file eostools.py.

Referenced by lfnToEOS().

54 
55 def isLFN( path ):
56  """Tests whether this path is a CMS LFN (name starts with /store...)"""
57  # return re.match('^/store.*', path ) is not None
58  return path.startswith('/store')
def isLFN
Definition: eostools.py:54
def eostools.lfnToEOS (   path)
Converts LFN to EOS.

If path is not an LFN in the first place, return path.
??? shouldn't we raise an exception?

Definition at line 105 of file eostools.py.

References isLFN().

Referenced by cat().

106 def lfnToEOS( path ):
107  """Converts LFN to EOS.
108 
109  If path is not an LFN in the first place, return path.
110  ??? shouldn't we raise an exception?"""
111  if isLFN(path):
112  pfn = 'root://eoscms.cern.ch//eos/cms/' + path
113  return pfn.replace('//store','/store')
114  else:
115  return path
116 
#also define an alias for backwards compatibility
def lfnToEOS
Definition: eostools.py:105
def isLFN
Definition: eostools.py:54
def eostools.lfnToPFN (   path,
  tfcProt = 'rfio' 
)
Converts an LFN to a PFN. For example:
/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21
->
root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms

This function only checks path, and does not access the storage system.
If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used.
Otherwise, is uses the default one. 

??? what is tfcprot? 

Definition at line 74 of file eostools.py.

Referenced by isEOSDir(), runEOSCommand(), and runXRDCommand().

74 
75 def lfnToPFN( path, tfcProt = 'rfio'):
76  """Converts an LFN to a PFN. For example:
77  /store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21
78  ->
79  root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms
80 
81  This function only checks path, and does not access the storage system.
82  If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used.
83  Otherwise, is uses the default one.
84 
85  ??? what is tfcprot? """
86 
87  if path.startswith("/store/"):
88  path = path.replace("/store/","root://eoscms.cern.ch//eos/cms/store/")
89  if path.startswith("/pnfs/psi.ch/cms/trivcat/"):
90  path = path.replace("/pnfs/psi.ch/cms/trivcat/","root://t3se01.psi.ch//")
91  #print "path to cmsFile():", path
92  entity = cmsIO.cmsFile( path, tfcProt )
93 # tokens = cmsIO.splitPFN(entity.pfn)
94  pfn = '%s://%s//%s/' % (entity.protocol,entity.host,entity.path)
95 
96  pfn = entity.pfn
97  if tfcProt == 'rfio' and \
98  entity.path.startswith("/eos/cms/") and \
99  str(entity.stat()).startswith("Error 3011: Unable to stat"):
100 
101  pfn.replace("/eos/cms","/castor/cern.ch/cms")
102  pfn.replace("eoscms","castorcms")
103  return pfn
104 
def lfnToPFN
Definition: eostools.py:74
def eostools.listFiles (   path,
  rec = False,
  full_info = False 
)
Provides a list of the specified directory

Definition at line 297 of file eostools.py.

298 def listFiles(path, rec = False, full_info = False):
299  """Provides a list of the specified directory
300  """
301  # -- listing on the local filesystem --
302  if os.path.isdir( path ):
303  if not rec:
304  # not recursive
305  return [ '/'.join([path,file]) for file in os.listdir( path )]
306  else:
307  # recursive, directories are put in the list first,
308  # followed by the list of all files in the directory tree
309  result = []
310  allFiles = []
311  for root,dirs,files in os.walk(path):
312  result.extend( [ '/'.join([root,dir]) for dir in dirs] )
313  allFiles.extend( [ '/'.join([root,file]) for file in files] )
314  result.extend(allFiles)
315  return result
316  # -- listing on EOS --
317  cmd = 'dirlist'
318  if rec:
319  cmd = 'dirlistrec'
320  files, _, _ = runXRDCommand(path, cmd)
321  result = []
322  for line in files.split('\n'):
323  tokens = [t for t in line.split() if t]
324  if tokens:
325  #convert to an LFN
326  # result.append(tuple(tokens))
327  #COLIN need same interface for eos and local fs
328  if full_info:
329  result.append( tokens)
330  else:
331  result.append( tokens[4] )
332  return result
def listFiles
Definition: eostools.py:297
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def runXRDCommand
Definition: eostools.py:20
def eostools.ls (   path,
  rec = False 
)
Provides a simple list of the specified directory, works on EOS and locally

Definition at line 346 of file eostools.py.

Referenced by FWPFCandidateDetailView.addClusters(), HcalLaserEventFilter2012.addEventString(), DisplayGeom.analyze(), PixelLumiDQM.beginLuminosityBlock(), FWBeamSpotProxyBuilder.build(), FWPFClusterRPZProxyBuilder.build(), FWPFClusterRPZUtils.buildRhoPhiClusterLineSet(), FWPFClusterRPZUtils.buildRhoZClusterLineSet(), evf::EvFDaqDirector.bumpFile(), L1TSync.certifyLSBlock(), CompositeLogicalTrajectoryFilter.CompositeLogicalTrajectoryFilter(), L1TSync.doFractionInSync(), L1TRate.endLuminosityBlock(), PixelLumiDQM.endLuminosityBlock(), RawEventFileWriterForBU.endOfLS(), RateBuffer.fill(), EcalDeadCellTriggerPrimitiveFilter.filter(), RawEventFileWriterForBU.finishFileWrite(), RateBuffer.getLSAlgoRate(), RateBuffer.getLSRate(), RateBuffer.getLSTechRate(), BeamSpotWorkflow.getNewRunList(), IdealZDCTrapezoid.initCorners(), IdealCastorTrapezoid.initCorners(), EcalDeadCellTriggerPrimitiveFilter.loadEventInfo(), FWBeamSpotProxyBuilder.localModelChanges(), ls_cert_type(), lsbs_cert(), copyAndRename.main(), copyFromCastor.main(), rename.main(), copyFiles.main(), main(), splitter.main(), checkRuns.main(), nlumis(), evf::EvFDaqDirector.preGlobalEndLumi(), AlpgenSource.readAlpgenEvent(), FedRawDataInputSource.readSupervisor(), evf::FastMonitoringService.reportLockWait(), lumi::TRGScalers2DB.retrieveData(), GenericTripletGenerator.run(), GenericPairGenerator.run(), BeamHaloPairGenerator.run(), fffnaming.runLumiPrefix(), fffnaming.runLumiPrefixFill(), MonitorElement.setLumi(), DIPLumiDetail.setOrigin(), DIPLumiSummary.setOrigin(), FWPFClusterRPZProxyBuilder.sharedBuild(), TT6CMNSubtractor.subtract_(), IteratedMedianCMNSubtractor.subtract_(), SimpleCosmicBONSeeder.triplets(), evf::EvFDaqDirector.updateFuLock(), DTTimeEvolutionHisto.updateTimeSlot(), and RawEventOutputModuleForBU< Consumer >.write().

347 def ls(path, rec = False):
348  """Provides a simple list of the specified directory, works on EOS and locally"""
349  return [eosToLFN(t) for t in listFiles(path, rec)]
def ls
Definition: eostools.py:346
def eosToLFN
Definition: eostools.py:63
def listFiles
Definition: eostools.py:297
def eostools.ls_EOS (   path,
  rec = False 
)
Provides a simple list of the specified directory, works on EOS only, but is faster than the xrd version

Definition at line 350 of file eostools.py.

Referenced by datasetNotEmpty().

351 def ls_EOS(path, rec = False):
352  """Provides a simple list of the specified directory, works on EOS only, but is faster than the xrd version"""
353  if rec:
354  stdout, _, ret = runEOSCommand(path,'find','-f')
355  return [eosToLFN(line) for line in stdout.split('\n') if line]
356  else:
357  stdout, _, ret = runEOSCommand(path,'ls')
358  lfn = eosToLFN(path)
359  return [os.path.join(lfn,line) for line in stdout.split('\n') if line]
def ls_EOS
Definition: eostools.py:350
def eosToLFN
Definition: eostools.py:63
def runEOSCommand
Definition: eostools.py:36
def eostools.matchingFiles (   path,
  regexp 
)
Return a list of files matching a regexp

Definition at line 516 of file eostools.py.

517 def matchingFiles( path, regexp):
518  """Return a list of files matching a regexp"""
519 
520  # print path, regexp
521  pattern = re.compile( regexp )
522  #files = ls_EOS(path)
523  files = ls(path)
524  # print files
525  return [f for f in files if pattern.match(os.path.basename(f)) is not None]
def ls
Definition: eostools.py:346
def matchingFiles
Definition: eostools.py:516
def eostools.mkdir (   path)
Create a directory, either on EOS or locally

Definition at line 248 of file eostools.py.

Referenced by L1GtVhdlWriter.analyze(), spu.create_dir(), dd_to_html(), evf::EvFDaqDirector.EvFDaqDirector(), MillePedeMonitor.init(), LaserSorter.LaserSorter(), L1GtVhdlWriterCore.makeFirmware(), pos::PixelConfigFile.makeNewVersion(), L1MuGlobalMuonTrigger.produce(), EgammaObjects.saveHistos(), VariablePlotter.setDir(), LaserSorter.streamFileName(), WatcherStreamFileReader.WatcherStreamFileReader(), and RawEventFileWriterForBU.writeJsds().

249 def mkdir(path):
250  """Create a directory, either on EOS or locally"""
251  # print 'mkdir', path
252  if isEOS( path ) or isLFN(path):
253  createEOSDir(path)
254  else:
255  # recursive directory creation (like mkdir -p)
256  os.makedirs(path)
257  return path
def isLFN
Definition: eostools.py:54
def mkdir
Definition: eostools.py:248
def isEOS
Definition: eostools.py:59
def createEOSDir
Definition: eostools.py:227
def eostools.move (   src,
  dest 
)
Move filename1 to filename2 locally to the same server

Definition at line 508 of file eostools.py.

Referenced by CompatibleDetToGroupAdder.add(), tracking::TempMeasurements.add(), TkGluedMeasurementDet::HitCollectorForFastMeasurements.add(), TrajectorySegmentBuilder.addGroup(), CosmicTrajectoryBuilder.AddHit(), TkGluedMeasurementDet::HitCollectorForRecHits.addProjected(), BaseCkfTrajectoryBuilder.addToResult(), GroupedCkfTrajectoryBuilder.advanceOneLayer(), MultiTrackValidator.analyze(), DTCCBConfig.appendConfigKey(), LHESource.beginRun(), DQMStore.book(), Basic2DGenericPFlowClusterizer.buildClusters(), PFlow2DClusterizerWithTime.buildClusters(), DetIdAssociator.buildMap(), SeedFromConsecutiveHitsCreator.buildSeed(), TrackProducerAlgorithm< reco::Track >.buildTrack(), TrackProducerAlgorithm< reco::GsfTrack >.buildTrack(), DTReadOutMapping.cacheMap(), XrdAdaptor::RequestManager.checkSourcesImpl(), DAClusterizerInZ_vect.clusterize(), PixelThresholdClusterizer.clusterizeDetUnit(), edm::LuminosityBlock.commit_(), edm::Run.commit_(), edm::Event.commit_aux(), CaloTowersCreationAlgo.convert(), edm.convert_handle(), edm::SharedResourcesRegistry.createAcquirer(), edm::SharedResourcesRegistry.createAcquirerForSourceDelayedReader(), edm::impl.createCommunicatorIfNeeded(), StandAloneMuonFilter.createDefaultTrajectory(), edm::OutputModuleCommunicatorT< T >.createIfNeeded(), ESDigitizer.createNoisyList(), edm::maker::ModuleHolderT< T >.createOutputModuleCommunicator(), CosmicTrajectoryBuilder.createStartingTrajectory(), CRackTrajectoryBuilder.createStartingTrajectory(), KinematicTree.currentProductionVertex(), edm::RecordInParentless< PROD >.do_it(), edm::RecordInParentfull< PROD >.do_it(), OutsideInMuonSeeder.doLayer(), EcalTBDigiProducer.EcalTBDigiProducer(), LHESource.endRun(), MeasurementDet.fastMeasurements(), edm::detail::NamedEventSelector.fill(), edm::RootOutputFile.fillBranches(), HcalGeometry.fillDetIds(), edm::RootFile.fillIndexIntoFile(), HPDNoiseGenerator.fillNoiseSignals(), TkStripMeasurementDet.filteredRecHits(), cms::SiPixelDigitizer.finalizeEvent(), LocalFileSystem.findCachePath(), V0Fitter.fitAll(), KFFittingSmoother.fitOne(), edm::ProvenanceAdaptor.fixProcessHistory(), edm::EventBase.get(), edm::eventsetup::EventSetupRecord.get(), edm::Event.get(), edm.getAnyPtr(), PileUpEventPrincipal.getByLabel(), edm::EventBase.getByLabel(), edm::RunBase.getByLabel(), edm::LuminosityBlock.getByLabel(), edm::LuminosityBlockBase.getByLabel(), edm::Run.getByLabel(), edm::Event.getByLabel(), edm::LuminosityBlock.getByToken(), edm::Run.getByToken(), edm::Event.getByToken(), edm::PrincipalGetAdapter.getManyByType(), edm.getWrapperBasePtr(), PixelForwardLayer.groupedCompatibleDetsV(), PixelForwardLayerPhase1.groupedCompatibleDetsV(), TECLayer.groupedCompatibleDetsV(), CompositeTECWedge.groupedCompatibleDetsV(), Phase2OTECRingedLayer.groupedCompatibleDetsV(), TIDRing.groupedCompatibleDetsV(), TIBRing.groupedCompatibleDetsV(), TBLayer.groupedCompatibleDetsV(), Phase2OTECRing.groupedCompatibleDetsV(), PixelBlade.groupedCompatibleDetsV(), TOBRod.groupedCompatibleDetsV(), Phase1PixelBlade.groupedCompatibleDetsV(), Phase2OTBarrelRod.groupedCompatibleDetsV(), CompositeTECPetal.groupedCompatibleDetsV(), LayerMeasurements.groupedMeasurements(), XrdAdaptor::RequestManager.handle(), XrdAdaptor::RequestManager::OpenHandler.HandleResponseWithHosts(), SeedingLayerSetsBuilder.hits(), RectangularEtaPhiTrackingRegion.hits(), MultiHitGeneratorFromChi2.hitSets(), edm::EventProcessor.init(), DTCCBConfig.initialize(), XrdAdaptor::RequestManager.initialize(), initializeDummyPSet(), PFEGammaAlgo.initializeProtoCands(), TransientInitialStateEstimator.innerState(), GeomDetCompatibilityChecker.isCompatible(), SeedingLayerSetsBuilder::LayerSpec.LayerSpec(), CkfTrajectoryBuilder.limitedCandidates(), edm.makeESHandleExceptionFactory(), edm.makeHandleExceptionFactory(), edm::RootFile.makeProvenanceReaderMaker(), TrajectoryFitter.makeVect(), TkPixelMeasurementDet.measurements(), DQMStore.mergeAndResetMEsLuminositySummaryCache(), DQMStore.mergeAndResetMEsRunSummaryCache(), edm::ProductHolderBase.mergeProduct(), edm::InputProductHolder.mergeProduct_(), edm::ProducedProductHolder.mergeProduct_(), edm::AliasProductHolder.mergeProduct_(), DetGroupMerger.mergeTwoLevels(), KinematicTree.movePointerToTheTop(), edm::soahelper::SoATupleHelper< I, Args >.moveToNew(), BaseCkfTrajectoryBuilder.moveToResult(), MuonAssociatorByHits.MuonAssociatorByHits(), MuonToSimAssociatorByHits.MuonToSimAssociatorByHits(), extstd::clone_ptr< T >.operator=(), edm::InputTag.operator=(), DetGroupMerger.orderAndMergeTwoLevels(), PixelTrackReconstruction.PixelTrackReconstruction(), TFWLiteSelectorBasic.Process(), edm::SubProcess.process(), edm::EventAuxiliaryHistoryProducer.produce(), SeedCombiner.produce(), evf::EvFFEDSelector.produce(), MuonReSeeder.produce(), edm::LogErrorHarvester.produce(), edm::TriggerResultInserter.produce(), JetCorrectorProducer< T >.produce(), TrackMCQuality.produce(), TrackAssociatorByPositionProducer.produce(), TrackAssociatorByChi2Producer.produce(), QuickTrackAssociatorByHitsProducer.produce(), PFEGammaProducer.produce(), TcdsRawToDigi.produce(), TrackAssociatorByHitsProducer.produce(), TrajectorySeedProducer.produce(), ChainedJetCorrectorProducer.produce(), MuonToTrackingParticleAssociatorEDProducer.produce(), cms::CkfTrackCandidateMakerBase.produceBase(), TkGluedMeasurementDet.projectOnGluedDet(), TempTrajectory.push(), edm::RunPrincipal.put(), edm::LuminosityBlockPrincipal.put(), edm::Run.put(), edm::Event.put(), edm::LuminosityBlock.put(), edm::EventPrincipal.put(), KfTrackProducerBase.putInEvt(), edm::EventPrincipal.putOnRead(), edm::Principal.putOrMerge(), edm::ProductHolderBase.putProduct(), edm::InputProductHolder.putProduct_(), edm::ProducedProductHolder.putProduct_(), edm::AliasProductHolder.putProduct_(), FedRawDataInputSource.read(), edm::StreamerInputSource.read(), edm::RootFile.readCurrentEvent(), LHESource.readEvent_(), edm::PoolSource.readFile_(), edm::EventPrincipal.readFromSource_(), GroupedCkfTrajectoryBuilder.rebuildSeedingRegion(), GroupedCkfTrajectoryBuilder.rebuildTrajectories(), TkStripMeasurementDet.recHits(), RecoTauDiscriminantCutMultiplexer.RecoTauDiscriminantCutMultiplexer(), TrajectorySegmentBuilder.redoMeasurements(), VertexFitterManager.registerFitter(), edm::RunPrincipal.resolveProductImmediate(), edm::LuminosityBlockPrincipal.resolveProductImmediate(), ESDigitizer.run(), CaloTDigitizer< Traits >.run(), PixelForwardLayer.searchNeighbors(), PixelForwardLayerPhase1.searchNeighbors(), TIBRing.searchNeighbors(), CRackTrajectoryBuilder.seedMeasurements(), edm::InputProductHolder.setProduct(), PFECALSuperClusterAlgo.setTokens(), HLTJets.setup(), edm::service::SiteLocalConfigService.SiteLocalConfigService(), DDValue::StringHolder.StringHolder(), edm::SubProcess.SubProcess(), TempTrajectory.TempTrajectory(), TrackingLayer.toIdString(), TrackingLayer.toString(), edm::service::Tracer.Tracer(), TrajectorySmoother.trajectories(), TrajectoryBuilder.trajectories(), TrajectorySeedProducer.TrajectorySeedProducer(), MuonTrajectoryUpdator.update(), TrajectorySegmentBuilder.updateCandidatesWithBestHit(), TrajectorySegmentBuilder.updateTrajectory(), and CkfTrajectoryBuilder.updateTrajectory().

509 def move(src, dest):
510  """Move filename1 to filename2 locally to the same server"""
511 
512  src = eosToLFN(src)
513  dest = eosToLFN(dest)
514 
515  runXRDCommand(src,'mv', lfnToEOS(dest))
def eosToLFN
Definition: eostools.py:63
def lfnToEOS
Definition: eostools.py:105
def move
Definition: eostools.py:508
def runXRDCommand
Definition: eostools.py:20
def eostools.remove (   files,
  rec = False 
)
Remove a list of files and directories, possibly recursively

Colin: Is that obsolete? why not use rm?

Definition at line 379 of file eostools.py.

380 def remove( files, rec = False):
381  """Remove a list of files and directories, possibly recursively
382 
383  Colin: Is that obsolete? why not use rm?"""
384  for path in files:
385  lfn = eosToLFN(path)
386  if not rec:
387  rm(path)
388  else:
389  #this should be used with care
390  file_list = ls(path, rec = True)
391  file_list.append(lfn)
392 
393  #order the files in depth order - i.e. remove the deepest files first
394  files_rec = sorted([(len([ff for ff in f.split('/') if ff]), f) for f in file_list if f and f.startswith(lfn)], reverse = True)
395 
396  for f in files_rec:
397  rm(f[1])
def ls
Definition: eostools.py:346
def remove
Definition: eostools.py:379
def eosToLFN
Definition: eostools.py:63
def rm
Definition: eostools.py:360
def eostools.rm (   path,
  rec = False 
)
rm, works on EOS and locally.

Colin: should implement a -f mode and a confirmation when deleting dirs recursively.

Definition at line 360 of file eostools.py.

Referenced by remove().

361 def rm(path, rec=False):
362  """rm, works on EOS and locally.
363 
364  Colin: should implement a -f mode and a confirmation when deleting dirs recursively."""
365  # print 'rm ', path
366  path = lfnToEOS(path)
367  if isEOS(path):
368  if rec:
369  runEOSCommand(path, 'rm', '-r')
370  else:
371  runEOSCommand(path,'rm')
372  elif os.path.exists(path):
373  if not rec:
374  os.remove( path )
375  else:
376  shutil.rmtree(path)
377  else:
378  raise ValueError(path + ' is not EOS and not local... should not happen!')
def lfnToEOS
Definition: eostools.py:105
def runEOSCommand
Definition: eostools.py:36
def rm
Definition: eostools.py:360
def isEOS
Definition: eostools.py:59
def eostools.runEOSCommand (   path,
  cmd,
  args 
)
Run an eos command.

!!! Will, when the EOS command fails, it passes silently...
I think we should really try and raise an exception in case of problems.
should be possible as the return code is provided in the tuple returned by runner.

Definition at line 36 of file eostools.py.

References eosToLFN(), and lfnToPFN().

Referenced by chmod(), createEOSDir(), eosDirSize(), ls_EOS(), and rm().

36 
37 def runEOSCommand(path, cmd, *args):
38  """Run an eos command.
39 
40  !!! Will, when the EOS command fails, it passes silently...
41  I think we should really try and raise an exception in case of problems.
42  should be possible as the return code is provided in the tuple returned by runner."""
43 
44  lfn = eosToLFN(path)
45  pfn = lfnToPFN(lfn)
46  tokens = cmsIO.splitPFN(pfn)
47 
48  #obviously, this is not nice
49  command = ['/afs/cern.ch/project/eos/installation/pro/bin/eos.select', cmd]
50  command.extend(args)
51  command.append(tokens[2])
52  runner = cmsIO.cmsFileManip()
53  return runner.runCommand(command)
def eosToLFN
Definition: eostools.py:63
def runEOSCommand
Definition: eostools.py:36
def lfnToPFN
Definition: eostools.py:74
def eostools.runXRDCommand (   path,
  cmd,
  args 
)
Run an xrd command.

!!! Will, what is happening in case of problem?
??? At some point, should return a list of lines instead of a string.

Definition at line 20 of file eostools.py.

References eosToLFN(), and lfnToPFN().

Referenced by cat(), isDirectory(), isFile(), listFiles(), and move().

20 
21 def runXRDCommand(path, cmd, *args):
22  """Run an xrd command.
23 
24  !!! Will, what is happening in case of problem?
25  ??? At some point, should return a list of lines instead of a string."""
26 
27  lfn = eosToLFN(path)
28  #print "lfn:", lfn, cmd
29  tokens = cmsIO.splitPFN(lfnToPFN(lfn))
30 
31  command = ['xrd', tokens[1], cmd, tokens[2]]
32  command.extend(args)
33  runner = cmsIO.cmsFileManip()
34  # print ' '.join(command)
35  return runner.runCommand(command)
def eosToLFN
Definition: eostools.py:63
def runXRDCommand
Definition: eostools.py:20
def lfnToPFN
Definition: eostools.py:74
def eostools.setCAFPath ( )
Hack to get the CAF scripts on the PYTHONPATH

Definition at line 11 of file eostools.py.

11 
12 def setCAFPath():
13  """Hack to get the CAF scripts on the PYTHONPATH"""
14  caf = '/afs/cern.ch/cms/caf/python'
15 
16  if caf not in sys.path:
17  sys.path.append(caf)
setCAFPath()
def setCAFPath
Definition: eostools.py:11
def eostools.which (   cmd)

Definition at line 333 of file eostools.py.

Referenced by OOTPileupCorrData.apply(), MuonResidualsFitter.computeHistogramRangeAndBinning(), gen::EvtGenLHCInterface.decay(), condPython.defineWhat< cond::ecalcond::Container >(), condPython.defineWhat< EcalDAQTowerStatus >(), condPython.defineWhat< EcalDCSTowerStatus >(), condPython.defineWhat< EcalDQMChannelStatus >(), condPython.defineWhat< EcalDQMTowerStatus >(), condPython.defineWhat< EcalPedestals >(), condPython.defineWhat< EcalPulseCovariances >(), condPython.defineWhat< EcalPulseShapes >(), condPython.defineWhat< Pedestals >(), condPython.defineWhat< RPCObGas >(), condPython.defineWhat< RPCObImon >(), condPython.defineWhat< RPCObTemp >(), condPython.defineWhat< RPCObVmon >(), pat::PATSingleVertexSelector.filter_(), ConstantStepOdeSolver.getCoordinate(), OOTPileupCorrData.getCorrectionByID(), HBHENegativeEFilter.getEtaIndex(), LMFDat.getNeighbour(), AbsElectronicODERHS.getParameter(), ConstantStepOdeSolver.getPeakTime(), ConstantStepOdeSolver.integrateCoordinate(), ConstantStepOdeSolver.interpolateCoordinate(), MuonResidualsTwoBin.mean(), PiecewiseScalingPolynomial.operator()(), MuonResidualsTwoBin.read(), InputTagDistributorService.retrieve(), TrackMultiSelector.select(), MuonResidualsFitter.selectPeakResiduals_simple(), AbsElectronicODERHS.setParameter(), MuonResidualsTwoBin.stdev(), ConstantStepOdeSolver.truncateCoordinate(), MuonResidualsTwoBin.wmean(), MuonResidualsTwoBin.write(), MuonResidualsFitter.write(), and ConstantStepOdeSolver.writeHistory().

334 def which(cmd):
335  command = ['which', cmd]
336  runner = cmsIO.cmsFileManip()
337  out, _, _ = runner.runCommand(command)
338 
339  lines = [line for line in out.split('\n') if line]
340  if len(lines) == 1:
341  return lines[0]
342  elif len(lines) == 2:
343  return lines[1]
344  else:
345  return lines
def which
Definition: eostools.py:333
def eostools.xrdcp (   src,
  dest 
)
Does a copy of files using xrd.

Colin: implement a generic cp interface as done for rm, ls, etc?

Definition at line 427 of file eostools.py.

428 def xrdcp(src, dest):
429  """Does a copy of files using xrd.
430 
431  Colin: implement a generic cp interface as done for rm, ls, etc?"""
432 
433  recursive = False
434 
435  #first the src file
436  pfn_src = src
437  if os.path.exists(src):
438  #local
439  pfn_src = src
440  if os.path.isdir(src):
441  recursive = True
442  elif fileExists(src):
443  src = eosToLFN(src)
444  pfn_src = lfnToPFN(src)
445  if isDirectory(src):
446  recursive = True
447  else:
448  raise ValueError(src + ' does not exist.')
449 
450  #now the dest
451  pfn_dest = dest
452  if isEOSDir(dest):
453  dest = eosToLFN(dest)
454  pfn_dest = lfnToPFN(dest)
455  if isDirectory(dest):
456  tokens = cmsIO.splitPFN(pfn_dest)
457  pfn_dest = '%s://%s//%s/' % (tokens[0],tokens[1],tokens[2])
458  elif os.path.exists(dest):
459  pfn_dest = dest
460 
461  command = ['xrdcp']
462  if recursive:
463  # print 'recursive'
464  topDir = src.rstrip('/').split('/')[-1]
465  if topDir != '.':
466  dest = '/'.join([dest, topDir])
467  # print 'mkdir ' + dest
468  mkdir( dest )
469  files = listFiles(src, rec=True)
470  # pprint.pprint( [file[4] for file in files] )
471  for srcFile in files:
472  # srcFile = file[4]
473  pfnSrcFile = srcFile
474  if isEOSDir(srcFile):
475  srcFile = eosToLFN(srcFile)
476  pfnSrcFile = lfnToPFN(srcFile)
477  destFile = srcFile.replace( src, '' )
478  destFile = '/'.join([dest,destFile])
479  pfnDestFile = destFile
480  if isEOSDir(destFile):
481  lfnDestFile = eosToLFN(destFile)
482  pfnDestFile = lfnToPFN(lfnDestFile)
483  # print 'srcFile', pfnSrcFile
484  # print 'destFile', pfnDestFile
485  if isFile(srcFile):
486  _xrdcpSingleFile( pfnSrcFile, pfnDestFile )
487  else:
488  mkdir(destFile)
489  else:
490  _xrdcpSingleFile( pfn_src, pfn_dest )
491 
def isDirectory
Definition: eostools.py:258
def fileExists
Definition: eostools.py:195
def eosToLFN
Definition: eostools.py:63
def listFiles
Definition: eostools.py:297
def isEOSDir
Definition: eostools.py:119
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def isFile
Definition: eostools.py:271
def xrdcp
Definition: eostools.py:427
def _xrdcpSingleFile
Definition: eostools.py:492
def mkdir
Definition: eostools.py:248
double split
Definition: MVATrainer.cc:139
def lfnToPFN
Definition: eostools.py:74

Variable Documentation

eostools.castorToLFN = eosToLFN

Definition at line 72 of file eostools.py.

eostools.createCastorDir = createEOSDir

Definition at line 246 of file eostools.py.

eostools.isCastorDir = isEOSDir

Definition at line 147 of file eostools.py.

eostools.isCastorFile = isEOSFile

Definition at line 192 of file eostools.py.

eostools.lfnToCastor = lfnToEOS

Definition at line 117 of file eostools.py.