CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
cmsLHEtoEOSManager Namespace Reference

Functions

def articleExist
 
def fileUpload
 
def findXrdDir
 
def lastArticle
 

Variables

string __version__ = "$Revision: 1.13 $"
 
string action = 'store_true'
 
string default = ''
 
string defaultEOScpCommand = 'xrdcp -np '
 
string defaultEOSfeCommand = 'xrd eoscms existfile '
 
string defaultEOSlistCommand = 'xrd eoscms dirlist '
 
string defaultEOSLoadPath = 'root://eoscms/'
 
string defaultEOSmkdirCommand = 'xrd eoscms mkdir '
 
string defaultEOSRootPath = '/eos/cms/store/lhe'
 
string dest = 'fileList'
 
tuple exeCheckIntegrity = subprocess.Popen(["/bin/sh","-c", theCheckIntegrityCommand])
 
tuple exeCheckSum = subprocess.Popen(["/afs/cern.ch/cms/caf/bin/cms_adler32",f], stdout=subprocess.PIPE)
 
tuple exeCompression = subprocess.Popen(["/bin/sh","-c",theCompressionCommand])
 
tuple exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand])
 
tuple exeList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
 
tuple exeUpload = subprocess.Popen(["/bin/sh","-c",theCommand])
 
tuple f = open(options.files_from)
 
tuple getCheckSum = subprocess.Popen(["awk", "{print $1}"], stdin=exeCheckSum.stdout, stdout=subprocess.PIPE)
 
string help = 'LHE local file list to be uploaded, separated by ","'
 
tuple intCode = exeCheckIntegrity.wait()
 
tuple l = l.strip()
 
string listPath = defaultEOSRootPath+'/'
 
int newArt = 0
 
tuple oldArt = lastArticle()
 
tuple parser = optparse.OptionParser(usage)
 
 reallyDoIt = notoptions.dryRun
 
string theCheckIntegrityCommand = 'xmllint -noout '
 
list theCheckSumList = []
 
string theCommand = defaultEOSmkdirCommand+' '
 
list theCompressedFilesList = []
 
string theCompressionCommand = 'xz '
 
list theList = []
 
string uploadPath = ''
 
string usage = 'cmsLHEtoEOSManager.py <options>'
 

Function Documentation

def cmsLHEtoEOSManager.articleExist (   artId)

Definition at line 27 of file cmsLHEtoEOSManager.py.

References findXrdDir().

27 
28 def articleExist(artId):
29 
30  itExists = False
31  theCommand = defaultEOSlistCommand+' '+defaultEOSRootPath
32  dirList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
33  for line in dirList.stdout.readlines():
34  if findXrdDir(line) == str(artId):
35  itExists = True
36 
37  return itExists
def cmsLHEtoEOSManager.fileUpload (   uploadPath,
  lheList,
  checkSumList,
  reallyDoIt 
)

Definition at line 54 of file cmsLHEtoEOSManager.py.

54 
55 def fileUpload(uploadPath,lheList, checkSumList, reallyDoIt):
56 
57  inUploadScript = ''
58  index = 0
59  for f in lheList:
60  realFileName = f.split('/')[-1]
61  # Check the file existence
62  newFileName = uploadPath+'/'+str(realFileName)
63  addFile = True
64  additionalOption = ''
65  theCommand = defaultEOSfeCommand+' '+newFileName
66  exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
67  result = exeFullList.stdout.readlines()
68  if result[0].rstrip('\n') == 'The file exists.':
69  addFile = False
70  print 'File '+newFileName+' already exists: do you want to overwrite? [y/n]'
71  reply = raw_input()
72  if reply == 'y' or reply == 'Y':
73  addFile = True
74  additionalOption = ' -f '
75  print ''
76  print 'Overwriting file '+newFileName+'\n'
77  # add the file
78  if addFile:
79 # print 'Adding file '+str(f)+'\n'
80  inUploadScript = defaultEOScpCommand + additionalOption + ' ' + str(f) + ' ' + defaultEOSLoadPath+uploadPath + '/' + str(realFileName)
81  print 'Uploading file %s...' % str(f)
82  if reallyDoIt:
83  exeRealUpload = subprocess.Popen(["/bin/sh","-c",inUploadScript])
84  exeRealUpload.communicate()
85  eosCheckSumCommand = '/afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select find --checksum ' + uploadPath + '/' + str(realFileName) + ' | awk \'{print $2}\' | cut -d= -f2'
86  exeEosCheckSum = subprocess.Popen(eosCheckSumCommand ,shell=True, stdout=subprocess.PIPE)
87  EosCheckSum = exeEosCheckSum.stdout.read()
88  assert exeEosCheckSum.wait() == 0
89  # print 'checksum: eos = ' + EosCheckSum + 'orig file = ' + checkSumList[index] + '\n'
90  if checkSumList[index] not in EosCheckSum:
91  print 'WARNING! The checksum for file ' + str(realFileName) + ' in EOS\n'
92  print EosCheckSum + '\n'
93  print 'does not match the checksum of the original one\n'
94  print checkSumList[index] + '\n'
95  print 'please try to re-upload file ' + str(realFileName) + ' to EOS.\n'
96  else:
97  print 'Checksum OK for file ' + str(realFileName)
98  index = index+1
99 
100 # launch the upload shell script
101 
102 # print '\n Launching upload script \n'+inUploadScript+'\n at '+time.asctime(time.localtime(time.time()))+' ...\n'
103 # if reallyDoIt:
104 # exeRealUpload = subprocess.Popen(["/bin/sh","-c",inUploadScript])
105 # exeRealUpload.communicate()
106  print '\n Upload ended at '+time.asctime(time.localtime(time.time()))
def cmsLHEtoEOSManager.findXrdDir (   theDirRecord)

Definition at line 19 of file cmsLHEtoEOSManager.py.

References split.

Referenced by articleExist(), and lastArticle().

19 
20 def findXrdDir(theDirRecord):
21 
22  elements = theDirRecord.split(' ')
23  if len(elements) > 1:
24  return elements[-1].rstrip('\n').split('/')[-1]
25  else:
26  return None
double split
Definition: MVATrainer.cc:139
def cmsLHEtoEOSManager.lastArticle ( )

Definition at line 38 of file cmsLHEtoEOSManager.py.

References findXrdDir(), and bookConverter.max.

38 
39 def lastArticle():
40 
41  artList = [0]
42 
43  theCommand = defaultEOSlistCommand+' '+defaultEOSRootPath
44  dirList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)
45  for line in dirList.stdout.readlines():
46  try:
47  if line.rstrip('\n') != '':
48  artList.append(int(findXrdDir(line)))
49  except:
50  break
51 
52  return max(artList)
53 

Variable Documentation

string cmsLHEtoEOSManager.__version__ = "$Revision: 1.13 $"

Definition at line 5 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.action = 'store_true'

Definition at line 126 of file cmsLHEtoEOSManager.py.

int cmsLHEtoEOSManager.default = ''

Definition at line 118 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOScpCommand = 'xrdcp -np '

Definition at line 17 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSfeCommand = 'xrd eoscms existfile '

Definition at line 16 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSlistCommand = 'xrd eoscms dirlist '

Definition at line 14 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSLoadPath = 'root://eoscms/'

Definition at line 13 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSmkdirCommand = 'xrd eoscms mkdir '

Definition at line 15 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.defaultEOSRootPath = '/eos/cms/store/lhe'

Definition at line 12 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.dest = 'fileList'

Definition at line 119 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeCheckIntegrity = subprocess.Popen(["/bin/sh","-c", theCheckIntegrityCommand])

Definition at line 213 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeCheckSum = subprocess.Popen(["/afs/cern.ch/cms/caf/bin/cms_adler32",f], stdout=subprocess.PIPE)

Definition at line 228 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeCompression = subprocess.Popen(["/bin/sh","-c",theCompressionCommand])

Definition at line 222 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeFullList = subprocess.Popen(["/bin/sh","-c",theCommand])

Definition at line 276 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeList = subprocess.Popen(["/bin/sh","-c",theCommand], stdout=subprocess.PIPE)

Definition at line 263 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.exeUpload = subprocess.Popen(["/bin/sh","-c",theCommand])

Definition at line 246 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.f = open(options.files_from)

Definition at line 188 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.getCheckSum = subprocess.Popen(["awk", "{print $1}"], stdin=exeCheckSum.stdout, stdout=subprocess.PIPE)

Definition at line 229 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.help = 'LHE local file list to be uploaded, separated by ","'

Definition at line 117 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.intCode = exeCheckIntegrity.wait()

Definition at line 214 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.l = l.strip()

Definition at line 192 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.listPath = defaultEOSRootPath+'/'

Definition at line 261 of file cmsLHEtoEOSManager.py.

int cmsLHEtoEOSManager.newArt = 0

Definition at line 234 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.oldArt = lastArticle()

Definition at line 240 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.parser = optparse.OptionParser(usage)

Definition at line 115 of file cmsLHEtoEOSManager.py.

cmsLHEtoEOSManager.reallyDoIt = notoptions.dryRun

Definition at line 162 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.theCheckIntegrityCommand = 'xmllint -noout '

Definition at line 212 of file cmsLHEtoEOSManager.py.

list cmsLHEtoEOSManager.theCheckSumList = []

Definition at line 198 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.theCommand = defaultEOSmkdirCommand+' '

Definition at line 244 of file cmsLHEtoEOSManager.py.

list cmsLHEtoEOSManager.theCompressedFilesList = []

Definition at line 197 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.theCompressionCommand = 'xz '

Definition at line 221 of file cmsLHEtoEOSManager.py.

tuple cmsLHEtoEOSManager.theList = []

Definition at line 182 of file cmsLHEtoEOSManager.py.

Referenced by MuonAlignment.recursiveList().

string cmsLHEtoEOSManager.uploadPath = ''

Definition at line 235 of file cmsLHEtoEOSManager.py.

string cmsLHEtoEOSManager.usage = 'cmsLHEtoEOSManager.py <options>'

Definition at line 114 of file cmsLHEtoEOSManager.py.