CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
cmsLHEtoEOSManager Namespace Reference

Functions

def articleExist
 
def fileUpload
 
def findXrdDir
 
def lastArticle
 

Variables

string __version__ = "$Revision: 1.12 $"
 
string action = 'store_true'
 
string default = ''
 
string defaultEOScpCommand = 'xrdcp -np '
 
string defaultEOSfeCommand = 'xrd eoscms existfile '
 
string defaultEOSlistCommand = 'xrd eoscms dirlist '
 
string defaultEOSLoadPath = 'root://eoscms/'
 
string defaultEOSmkdirCommand = 'xrd eoscms mkdir '
 
string defaultEOSRootPath = '/eos/cms/store/lhe'
 
string dest = 'fileList'
 
tuple exeCheckIntegrity = subprocess.Popen(["/bin/sh","-c", theCheckIntegrityCommand])
 
tuple exeCheckSum = subprocess.Popen(["/afs/cern.ch/cms/caf/bin/cms_adler32",f], stdout=subprocess.PIPE)
 
tuple exeCompression = subprocess.Popen(["/bin/sh","-c",theCompressionCommand])
 
tuple exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand])
 
tuple exeList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
 
tuple exeUpload = subprocess.Popen(["/bin/sh","-c",theCommand])
 
tuple f = open(options.files_from)
 
tuple getCheckSum = subprocess.Popen(["awk", "{print $1}"], stdin=exeCheckSum.stdout, stdout=subprocess.PIPE)
 
string help = 'LHE local file list to be uploaded, separated by ","'
 
tuple intCode = exeCheckIntegrity.wait()
 
tuple l = l.strip()
 
string listPath = defaultEOSRootPath+'/'
 
int newArt = 0
 
tuple oldArt = lastArticle()
 
tuple parser = optparse.OptionParser(usage)
 
 reallyDoIt = notoptions.dryRun
 
string theCheckIntegrityCommand = 'xmllint -noout '
 
list theCheckSumList = []
 
string theCommand = defaultEOSmkdirCommand+' '
 
list theCompressedFilesList = []
 
string theCompressionCommand = 'xz '
 
list theList = []
 
string uploadPath = ''
 
string usage = 'cmsLHEtoEOSManager.py <options>'
 

Function Documentation

def cmsLHEtoEOSManager.articleExist (   artId)

Definition at line 25 of file cmsLHEtoEOSManager.py.

References findXrdDir().

25 
26 def articleExist(artId):
27 
28  itExists = False
29  theCommand = defaultEOSlistCommand+' '+defaultEOSRootPath
30  dirList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
31  for line in dirList.stdout.readlines():
32  if findXrdDir(line) == str(artId):
33  itExists = True
34 
35  return itExists
def cmsLHEtoEOSManager.fileUpload (   uploadPath,
  lheList,
  checkSumList,
  reallyDoIt 
)

Definition at line 52 of file cmsLHEtoEOSManager.py.

52 
53 def fileUpload(uploadPath,lheList, checkSumList, reallyDoIt):
54 
55  inUploadScript = ''
56  index = 0
57  for f in lheList:
58  realFileName = f.split('/')[-1]
59  # Check the file existence
60  newFileName = uploadPath+'/'+str(realFileName)
61  addFile = True
62  additionalOption = ''
63  theCommand = defaultEOSfeCommand+' '+newFileName
64  exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
65  result = exeFullList.stdout.readlines()
66  if result[0].rstrip('\n') == 'The file exists.':
67  addFile = False
68  print 'File '+newFileName+' already exists: do you want to overwrite? [y/n]'
69  reply = raw_input()
70  if reply == 'y' or reply == 'Y':
71  addFile = True
72  additionalOption = ' -f '
73  print ''
74  print 'Overwriting file '+newFileName+'\n'
75  # add the file
76  if addFile:
77 # print 'Adding file '+str(f)+'\n'
78  inUploadScript = defaultEOScpCommand + additionalOption + ' ' + str(f) + ' ' + defaultEOSLoadPath+uploadPath + '/' + str(realFileName)
79  print 'Uploading file %s...' % str(f)
80  if reallyDoIt:
81  exeRealUpload = subprocess.Popen(["/bin/sh","-c",inUploadScript])
82  exeRealUpload.communicate()
83  eosCheckSumCommand = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select find --checksum ' + uploadPath + '/' + str(realFileName) + ' | awk \'{print $2}\' | cut -d= -f2'
84  exeEosCheckSum = subprocess.Popen(eosCheckSumCommand ,shell=True, stdout=subprocess.PIPE)
85  EosCheckSum = exeEosCheckSum.stdout.read()
86  assert exeEosCheckSum.wait() == 0
87  print 'checksum: eos = ' + EosCheckSum + 'orig file = ' + checkSumList[index] + '\n'
88  if checkSumList[index] not in EosCheckSum:
89  print 'WARNING! The checksum for file ' + str(realFileName) + ' in EOS\n'
90  print EosCheckSum + '\n'
91  print 'does not match the checksum of the original one\n'
92  print checkSumList[index] + '\n'
93  print 'please try to re-upload file ' + str(realFileName) + ' to EOS.\n'
94  index = index+1
95 
96 # launch the upload shell script
97 
98 # print '\n Launching upload script \n'+inUploadScript+'\n at '+time.asctime(time.localtime(time.time()))+' ...\n'
99 # if reallyDoIt:
100 # exeRealUpload = subprocess.Popen(["/bin/sh","-c",inUploadScript])
101 # exeRealUpload.communicate()
102  print '\n Upload ended at '+time.asctime(time.localtime(time.time()))
def cmsLHEtoEOSManager.findXrdDir (   theDirRecord)

Definition at line 17 of file cmsLHEtoEOSManager.py.

References split.

Referenced by articleExist(), and lastArticle().

17 
18 def findXrdDir(theDirRecord):
19 
20  elements = theDirRecord.split(' ')
21  if len(elements) > 1:
22  return elements[-1].rstrip('\n').split('/')[-1]
23  else:
24  return None
double split
Definition: MVATrainer.cc:139
def cmsLHEtoEOSManager.lastArticle ( )

Definition at line 36 of file cmsLHEtoEOSManager.py.

References findXrdDir(), and bookConverter.max.

36 
37 def lastArticle():
38 
39  artList = [0]
40 
41  theCommand = defaultEOSlistCommand+' '+defaultEOSRootPath
42  dirList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
43  for line in dirList.stdout.readlines():
44  try:
45  if line.rstrip('\n') != '':
46  artList.append(int(findXrdDir(line)))
47  except:
48  break
49 
50  return max(artList)
51 

Variable Documentation

string cmsLHEtoEOSManager.__version__ = "$Revision: 1.12 $"

Definition at line 3 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.action = 'store_true'

Definition at line 122 of file cmsLHEtoEOSManager.py.

int cmsLHEtoEOSManager.default = ''

Definition at line 114 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOScpCommand = 'xrdcp -np '

Definition at line 15 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSfeCommand = 'xrd eoscms existfile '

Definition at line 14 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSlistCommand = 'xrd eoscms dirlist '

Definition at line 12 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSLoadPath = 'root://eoscms/'

Definition at line 11 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSmkdirCommand = 'xrd eoscms mkdir '

Definition at line 13 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSRootPath = '/eos/cms/store/lhe'

Definition at line 10 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.dest = 'fileList'

Definition at line 115 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeCheckIntegrity = subprocess.Popen(["/bin/sh","-c", theCheckIntegrityCommand])

Definition at line 215 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeCheckSum = subprocess.Popen(["/afs/cern.ch/cms/caf/bin/cms_adler32",f], stdout=subprocess.PIPE)

Definition at line 204 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeCompression = subprocess.Popen(["/bin/sh","-c",theCompressionCommand])

Definition at line 224 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand])

Definition at line 278 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)

Definition at line 265 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeUpload = subprocess.Popen(["/bin/sh","-c",theCommand])

Definition at line 248 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.f = open(options.files_from)

Definition at line 184 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.getCheckSum = subprocess.Popen(["awk", "{print $1}"], stdin=exeCheckSum.stdout, stdout=subprocess.PIPE)

Definition at line 205 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.help = 'LHE local file list to be uploaded, separated by ","'

Definition at line 113 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.intCode = exeCheckIntegrity.wait()

Definition at line 216 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.l = l.strip()

Definition at line 188 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.listPath = defaultEOSRootPath+'/'

Definition at line 263 of file cmsLHEtoEOSManager.py.

int cmsLHEtoEOSManager.newArt = 0

Definition at line 236 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.oldArt = lastArticle()

Definition at line 242 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.parser = optparse.OptionParser(usage)

Definition at line 111 of file cmsLHEtoEOSManager.py.

cmsLHEtoEOSManager.reallyDoIt = notoptions.dryRun

Definition at line 158 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.theCheckIntegrityCommand = 'xmllint -noout '

Definition at line 214 of file cmsLHEtoEOSManager.py.

list cmsLHEtoEOSManager.theCheckSumList = []

Definition at line 194 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.theCommand = defaultEOSmkdirCommand+' '

Definition at line 246 of file cmsLHEtoEOSManager.py.

list cmsLHEtoEOSManager.theCompressedFilesList = []

Definition at line 193 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.theCompressionCommand = 'xz '

Definition at line 223 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.theList = []

Definition at line 178 of file cmsLHEtoEOSManager.py.

Referenced by MuonAlignment.recursiveList().

string cmsLHEtoEOSManager.uploadPath = ''

Definition at line 237 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.usage = 'cmsLHEtoEOSManager.py <options>'

Definition at line 110 of file cmsLHEtoEOSManager.py.