CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
archive Namespace Reference

Functions

def CheckCommand
 
def CheckFileStatus
 
def CheckPath
 
def CheckZippedFiles
 
def Cleaner
 
def ConfirmPath
 
def Delete
 
def DiskUsage
 
def filereg
 
def fileunreg
 
def GetAllFiles
 
def GetFileFromDB
 
def GetListOfFiles
 
def GetZippedFile
 
def RemoveAndRegister
 
def ScanDir
 
def SetPath
 
def TransferWithT0System
 

Variables

string arcdir = "/nfshome0/smaruyam/CMSSW_2_0_10/src/test/"
 
string bakdb = dbdir+"tmp/backup.db"
 
string cfgarg = " --config "
 
string cfgfile = " /nfshome0/smaruyam/CMSSW_2_0_10/src/test/myconfig.txt "
 
string db = dbdir+"db.db"
 
string dbdir = "/nfshome0/smaruyam/CMSSW_2_0_10/src/test/"
 
string dir = "/nfshome0/smaruyam/CMSSW_2_0_10/src/test/"
 
int disk_threshold = 80
 
string emptyString = "empty"
 
 EnableFileRemoval = False
 
 EnableTransfer = False
 
 filepath = outputFileName
 
int fileSizeThreshold = 1000000000
 
string firstFile = "DQM_Online_"
 
 flag = False
 
string fullTransferArg = cfgarg+" --type dqm --hostname srv-C2D05-19 --lumisection 1 --appname CMSSW --appversion CMSSW_2_0_10 "
 
tuple lastFile = zipFileList.split()
 
tuple logfile = open('archival_log.txt', 'a')
 
string outputFileName = arcdir+firstFile+lastFile+".zip"
 
 PathReplace = False
 
string sqlite3 = "sqlite3 "
 
string statusCheck = cfgarg+" --check --filename "
 
string targetdir = "/castor/cern.ch/cms/store/dqm/"
 
string tmpdb = dbdir+"tmp/tmp.db"
 
tuple transfer = TransferWithT0System(filepath,flag, logfile)
 
string transferScript = "/nfshome0/tier0/scripts/injectFileIntoTransferSystem.pl"
 
tuple zip = zipfile.ZipFile(outputFileName, "w")
 
tuple zipFileList = GetListOfFiles(logfile)
 
tuple zipFileSize = os.path.getsize(filepath)
 

Function Documentation

def archive.CheckCommand (   cmd,
  logfile 
)

Definition at line 90 of file archive.py.

Referenced by CheckFileStatus(), ConfirmPath(), GetAllFiles(), GetFileFromDB(), GetZippedFile(), ScanDir(), and TransferWithT0System().

90 
91 def CheckCommand(cmd, logfile):
92  result = commands.getstatusoutput(cmd)
93  if result[0] == 0:
94  output = result[1]
95  return result
96  else :
97  logfile.write("Command Exits with non-zero Status," + str(result[0]) + " Error = " + result[1] + "\n")
98  return result
99 
100 # generic function over
101 #
102 
103 #
104 # disk use check
105 """
106 Disk Usage Check
107 Reference to Cleaner()
108 df out put is assumed as follows.
109 Filesystem Size Used Avail Use% Mounted on
110 /dev/sda3 73G 45G 25G 65% /
111 /dev/sda1 99M 12M 83M 12% /boot
112 none 2.0G 0 2.0G 0% /dev/shm
113 /dev/sdb1 917G 83G 788G 10% /data
114 cmsnfshome0:/nfshome0
115  805G 673G 133G 84% /cmsnfshome0/nfshome0
"""
archive.CheckCommand
def CheckCommand
Definition: archive.py:90
def archive.CheckFileStatus (   filepath,
  logfile 
)

Definition at line 250 of file archive.py.

References CheckCommand(), split, and TransferWithT0System().

Referenced by Cleaner().

250 
251 def CheckFileStatus(filepath, logfile):
252  filename = filepath[len(arcdir):]
253  checkString = statusCheck + filename
254  mycmd = transferScript
255  myarg = checkString
256  cmd = mycmd + myarg
257  result = CheckCommand(cmd, logfile)
258  if result[0] == 0:
259  output = result[1].split('\n')
260  for line in output:
261  if line.find("FILES_TRANS_CHECKED: File found in database and checked by T0 system.") != -1: return True# file transferred successfully!
262  elif line.find("File not found in database.") != -1:# file not transferred at all
263  flag = False
264  TransferWithT0System(filepath,flag, logfile)
265  return False
266  elif line.find("FILES_INJECTED : File found in database and handed over to T0 system. ") != -1:# file must be transferred
267  flag = True
268  TransferWithT0System(filepath,flag, logfile)
269  mtime = os.stat(filepath).st_mtime
270  logfile.write("Old M Time is " + mtime + "\n")
271  os.utime(filepath,None)# change mtime to help path search
272  mtime2 = os.stat(filepath).st_mtime
273  logfile.write("New M Time is " + mtime2 + "\n")
274  logfile.write("File transfer need more time, please wait!\n")
275  return False
276 
277 # T0 transfer over
278 #
279 
280 #
281 # read file list from db
282 """
283 Get List of un-merged files, for Zipping
284 Reference to GetFileFromDB(), GetZippedFile()
"""
archive.TransferWithT0System
def TransferWithT0System
Definition: archive.py:224
archive.CheckCommand
def CheckCommand
Definition: archive.py:90
split
double split
Definition: MVATrainer.cc:139
archive.CheckFileStatus
def CheckFileStatus
Definition: archive.py:250
def archive.CheckPath (   filename,
  logfile 
)

Definition at line 155 of file archive.py.

References ConfirmPath(), and ScanDir().

Referenced by SetPath().

155 
156 def CheckPath(filename, logfile) :
157  mtime = os.stat(filename).st_mtime
158  year = time.localtime(mtime)[0]
159  month = time.localtime(mtime)[1]
160  if month > 9: yearmonth = str(year) + str(month)
161  else: yearmonth = str(year) + "0" + str(month)
162  path = targetdir + yearmonth
163  logfile.write("Best Guess for the path is " + path + "\n")# guess the path based on mtime
164  newpath = ConfirmPath(filename, path, logfile)# check if the path is correct
165  if cmp(newpath,emptyString) != 0 : return newpath
166  else :# scan all path, if the guess is wrong
167  newpath = ScanDir(filename, logfile)
168  return newpath
169 
170 """
171 Check File Path on Tape
172 Reference to CheckCommand()
"""
archive.ConfirmPath
def ConfirmPath
Definition: archive.py:173
archive.CheckPath
def CheckPath
Definition: archive.py:155
archive.ScanDir
def ScanDir
Definition: archive.py:197
def archive.CheckZippedFiles (   file,
  logfile 
)

Definition at line 420 of file archive.py.

References Delete(), GetZippedFile(), and split.

Referenced by Cleaner().

420 
421 def CheckZippedFiles(file, logfile):
422  logfile.write(" *** Check Zipped File ***\n")
423  flag = False
424  mergedfiles = GetZippedFile(logfile,flag).split("\n")
425  if len(mergedfiles) > 0:
426  for thisfile in mergedfiles:
427  if thisfile.find("zip") != -1 and cmp(thisfile,"") != 0 and cmp(thisfile,emptyString) != 0:
428  zip = zipfile.ZipFile(thisfile, "r")# open file to see it readable
429  for info in zip.infolist():# to see zipfile is uncompressed
430  if cmp(info.filename, file) == 0:
431  Delete(file,logfile)
432  return True
433  logfile.write("This file hasn't been zipped, " + file + " It shouldn't be deleted now!\n")
434 
435 """
436 Remove and Register Files
437 Reference to Delete(), filereg()
"""
archive.CheckZippedFiles
def CheckZippedFiles
Definition: archive.py:420
archive.Delete
def Delete
Definition: archive.py:449
archive.GetZippedFile
def GetZippedFile
Definition: archive.py:361
split
double split
Definition: MVATrainer.cc:139
def archive.Cleaner (   logfile)

Definition at line 399 of file archive.py.

References CheckFileStatus(), CheckZippedFiles(), Delete(), GetAllFiles(), and SetPath().

Referenced by DiskUsage().

399 
400 def Cleaner(logfile) :
401  logfile.write(" *** Cleaning File ***\n")
402  files = GetAllFiles(logfile)
403  for file in files:
404  if file.find(".zip") != -1:#zip file
405  status = CheckFileStatus(file, logfile)# check transfer status
406  if status is True and PathReplace is True:# remove file and replace the place
407  pathfind = SetPath(file, logfile)
408  if pathfind is True :# path found on tape
409  Delete(file, logfile)# remove only if transferred
410  return # exits when the files deleted
411  if file.find(".root") != -1 and file.find(dir) != -1:# Select Per-Run files
412  if PathReplace is False: CheckZippedFiles(file, logfile)# need check if zipped or not
413  if PathReplace is True: Delete(file, logfile)# must be zipped by this step
414  return # exits when the file deleted. ie, delete only one file
415  else : logfile.write("No File to be removed!\n")
416 
417 """
418 Remove File if zipped
419 Reference to Delete()
"""
archive.CheckZippedFiles
def CheckZippedFiles
Definition: archive.py:420
archive.Delete
def Delete
Definition: archive.py:449
archive.SetPath
def SetPath
Definition: archive.py:141
archive.GetAllFiles
def GetAllFiles
Definition: archive.py:378
archive.Cleaner
def Cleaner
Definition: archive.py:399
archive.CheckFileStatus
def CheckFileStatus
Definition: archive.py:250
def archive.ConfirmPath (   file,
  path,
  logfile 
)

Definition at line 173 of file archive.py.

References CheckCommand().

Referenced by CheckPath(), and ScanDir().

173 
174 def ConfirmPath(file, path, logfile) :
175  logfile.write(" *** Checking File Path ***\n ")
176  time.sleep(10)
177  fullpath = path + "/" + file[len(arcdir):]
178  mycmd = "rfdir "
179  myarg = fullpath
180  cmd = mycmd + myarg
181  result = CheckCommand(cmd, logfile)
182  if result[0] == 0:
183  output = result[1]
184  if cmp(output,"") != 0:
185  for line in output.split("\n"):
186  error_check = "No such file or directory"
187  if line.find(error_check) != -1 :return emptyString
188  logfile.write(" rfdir result is " + line + "\n")
189  if len(line.split()) > 7:
190  string = line.split()[-1]
191  if cmp(string,fullpath) == 0: return fullpath
192  return emptyString
193 
194 """
195 Scan Castor Directories
196 Reference to ConfirmPath(), CheckCommand()
"""
archive.CheckCommand
def CheckCommand
Definition: archive.py:90
archive.ConfirmPath
def ConfirmPath
Definition: archive.py:173
def archive.Delete (   file,
  logfile 
)

Definition at line 449 of file archive.py.

References DiskUsage(), and fileunreg().

Referenced by CheckZippedFiles(), Cleaner(), RemoveAndRegister(), and ora.setTableAccessPermission().

449 
450 def Delete(file, logfile):
451  fileunreg(db,bakdb,tmpdb,file,logfile)
452  logfile.write(file + "removed from db...\n")
453  os.remove(file)
454  logfile.write(file + "removed from disk...\n")
455 
456 # removal over
457 #
458 
459 #
460 # main program
461 """
462 Main Prog
463 Reference to DiskUsage(), GetListOfFiles(), TransferWithT0System(), RemoveAndRegister()
"""
archive.Delete
def Delete
Definition: archive.py:449
archive.fileunreg
def fileunreg
Definition: archive.py:65
def archive.DiskUsage (   logfile)

Definition at line 116 of file archive.py.

References Cleaner(), and split.

Referenced by Delete().

116 
117 def DiskUsage(logfile) :
118  logfile.write(" *** Checking Disk Usage ***\n")
119  df_file=os.popen('df')
120  usage = False
121  lines = df_file.readlines()
122  list = lines[4].split() # 5th line from top. Split at tab or white space
123  string = list[4][:-1] # NEED check for the host
124  fusage = float(string)
125  if fusage > disk_threshold : # disk is more than 80% full
126  logfile.write("Disk Usage too high = " + string + "%\n")
127  usage = True
128  if usage == True :
129  Cleaner(logfile)
130  else :
131  logfile.write("Disk Usage is low enough = " + string + "%\n")
132 
133 # disk use check over
134 #
135 
136 #
137 # Confirm the path to Transferred file on Castor
138 """
139 Set Path to Castor
140 Reference to CheckPath(), filereg()
"""
archive.DiskUsage
def DiskUsage
Definition: archive.py:116
archive.Cleaner
def Cleaner
Definition: archive.py:399
split
double split
Definition: MVATrainer.cc:139
def archive.filereg (   db,
  bakdb,
  tmpdb,
  file,
  logfile 
)

Definition at line 49 of file archive.py.

References SiPixelLorentzAngle_cfi.read.

Referenced by RemoveAndRegister(), and SetPath().

49 
50 def filereg(db,bakdb,tmpdb,file,logfile):
51  if os.path.exists(tmpdb): os.remove(tmpdb)
52  shutil.copy(db,tmpdb)
53  logfile.write('*** File Register ***\n')
54  logfile.write(os.popen('visDQMRegisterFile '+ tmpdb +' "/Global/Online/ALL" "Global run" '+ file).read())
55  t = datetime.now()
56  tstamp = t.strftime("%Y%m%d")
57  a = glob.glob(bakdb+'.'+tstamp+'*');
58  if not len(a):
59  tstamp = t.strftime("%Y%m%d_%H%M%S")
60  bakdb = bakdb+'.'+tstamp
61  shutil.copy(tmpdb,bakdb)
62  shutil.move(tmpdb,db)
63  else:
64  shutil.move(tmpdb,db)
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7
archive.filereg
def filereg
Definition: archive.py:49
def archive.fileunreg (   db,
  bakdb,
  tmpdb,
  oldfile,
  logfile 
)

Definition at line 65 of file archive.py.

References SiPixelLorentzAngle_cfi.read.

Referenced by Delete().

65 
66 def fileunreg(db,bakdb,tmpdb,oldfile,logfile):
67  if os.path.exists(tmpdb): os.remove(tmpdb)
68  shutil.copy(db,tmpdb)
69  logfile.write('*** File UnRegister ***\n')
70  logfile.write(os.popen('visDQMUnregisterFile '+ tmpdb +' ' + oldfile).read())
71  t = datetime.now()
72  tstamp = t.strftime("%Y%m%d")
73  a = glob.glob(bakdb+'.'+tstamp+'*');
74  if not len(a):
75  tstamp = t.strftime("%Y%m%d_%H%M%S")
76  bakdb = bakdb+'.'+tstamp
77  shutil.copy(tmpdb,bakdb)
78  shutil.move(tmpdb,db)
79  else:
80  shutil.move(tmpdb,db)
81 
82 # file register and un-register over
83 #
84 
85 #
86 # generic function
87 # check command exist status and retrive output messsage
88 """
89 Check and Return Output
"""
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7
archive.fileunreg
def fileunreg
Definition: archive.py:65
def archive.GetAllFiles (   logfile)

Definition at line 378 of file archive.py.

References CheckCommand(), and split.

Referenced by Cleaner().

378 
379 def GetAllFiles(logfile) :
380  logfile.write(" *** Getting All Files from db ***\n")
381  sqlite = db + " \"select name from t_files where name like '%DQM%.root' or name like '%DQM%.zip'order by mtime asc\""
382  mycmd = sqlite3
383  myarg = sqlite
384  cmd = mycmd + myarg
385  result = CheckCommand(cmd, logfile)
386  if result[0] == 0:
387  output = result[1].split('\n')
388  return output
389  else : return emptyString
390 
391 # file list over
392 #
393 
394 #
395 # remove files and register/unregister if needed
396 """
397 File Cleaner, Remove the oldest file
398 Reference to GetAllFiles(), CheckFileStatus(), SetPath(), Delete(), CheckZippedFiles()
"""
archive.CheckCommand
def CheckCommand
Definition: archive.py:90
archive.GetAllFiles
def GetAllFiles
Definition: archive.py:378
split
double split
Definition: MVATrainer.cc:139
def archive.GetFileFromDB (   logfile)

Definition at line 341 of file archive.py.

References CheckCommand().

Referenced by GetListOfFiles().

341 
342 def GetFileFromDB(logfile):
343  logfile.write(" *** Getting Per-Run File List from Master DB ***\n")
344  string = "'%DQM_V%_R%.root'"
345  search1 = "'%RPC%'"
346  search2 = "'%zip%'"
347  sqlite = " %s \"select name, size from t_files where name like %s and not name like %s and not name like %s order by mtime asc\" " %(db, string, search1, search2)
348  mycmd = sqlite3
349  myarg = sqlite
350  cmd = mycmd + myarg
351  result = CheckCommand(cmd, logfile)
352  if result[0] == 0:
353  return result[1]
354  else:
355  logfile.write(result[1])
356  return emptyString
357 
358 """
359 Get the last merged File, for Zipping
360 Reference to CheckCommand()
"""
archive.CheckCommand
def CheckCommand
Definition: archive.py:90
archive.GetFileFromDB
def GetFileFromDB
Definition: archive.py:341
def archive.GetListOfFiles (   logfile)

Definition at line 285 of file archive.py.

References GetFileFromDB(), GetZippedFile(), and split.

285 
286 def GetListOfFiles(logfile):
287  logfile.write("Retrieving list of files from DB ...\n")
288  totalSize = 0
289  zipFileList = ''
290  if PathReplace is True:# file removal is involved
291  fileList = GetFileFromDB(logfile).split('\n')
292  for line in fileList:
293  if cmp(line,"") != 0 and cmp(line,emptyString) != 0:
294  string = line.rstrip().split('|')
295  name = string[0]
296  logfile.write("String just read is " + string + "\n")
297  number = string[1]
298  logfile.write("Number just read is " + number + "\n")
299  totalSize += int(number)
300  logfile.write("Current File Size Sum is " + str(totalSize) + " out of Limit" + str(fileSizeThreshold) + "\n")
301  zipFileList += " " + name
302  if totalSize > fileSizeThreshold:
303  return zipFileList
304  if PathReplace is False:# file removal is NOT involved
305  activate = False
306  lastfile = ""
307  flag = True
308  mergedfiles = GetZippedFile(logfile,flag).split("\n")
309  if len(mergedfiles) > 0:
310  if cmp(mergedfiles[0],"") != 0 and cmp(mergedfiles[0],emptyString) != 0:
311  lastfile = mergedfiles[0]
312  logfile.write("Last Merged Zip File is " + lastfile + "\n")
313  elif cmp(mergedfiles[0],"") == 0:
314  activate = True
315  logfile.write("No Merged Zip File \n")
316  if len(mergedfiles) == 0:
317  activate = True
318  logfile.write("No Merged Zip File \n")
319  fileList = GetFileFromDB(logfile).split("\n")
320  for line in fileList:
321  if cmp(line,"") != 0 and cmp(line,emptyString) != 0:
322  string = line.split('|')
323  name = string[0]
324  if activate is True:
325  logfile.write("Name just read is " + name + "\n")
326  number = string[1]
327  logfile.write("Number just read is " + number + "\n")
328  totalSize += int(number)
329  logfile.write("Current File Size Sum is " + str(totalSize) + " out of Limit" + str(fileSizeThreshold) + "\n")
330  zipFileList += " " + name
331  if totalSize > fileSizeThreshold:
332  return zipFileList
333  if activate is False and cmp(lastfile,"") !=0:
334  if cmp(lastfile[len(arcdir)+len("DQM_Online_R000064821_"):-len(".zip")],name[len(dir)+len("DQM_V0001_R000064821_"):-len(".root")]) == 0:
335  activate = True
336  return emptyString # it's too small
337 
338 """
339 Read and sort file from db, for Zipping
340 Reference to CheckCommand()
"""
archive.GetFileFromDB
def GetFileFromDB
Definition: archive.py:341
archive.GetZippedFile
def GetZippedFile
Definition: archive.py:361
archive.GetListOfFiles
def GetListOfFiles
Definition: archive.py:285
split
double split
Definition: MVATrainer.cc:139
def archive.GetZippedFile (   logfile,
  flag 
)

Definition at line 361 of file archive.py.

References CheckCommand().

Referenced by CheckZippedFiles(), and GetListOfFiles().

361 
362 def GetZippedFile(logfile, flag):
363  logfile.write(" *** Getting Zipped File List from Master DB ***\n")
364  string = "'%DQM%.zip'"
365  if flag is True: sqlite = " %s \"select name from t_files where name like %s order by mtime desc\" " %(db, string)
366  if flag is False: sqlite = " %s \"select name from t_files where name like %s order by mtime asc\" " %(db, string)
367  mycmd = sqlite3
368  myarg = sqlite
369  cmd = mycmd + myarg
370  result = CheckCommand(cmd, logfile)
371  if result[0] == 0:
372  return result[1]
373  else: return emptyString
374 
375 """
376 Getting All Files from DB, for File Removal
377 Reference to CheckCommand()
"""
archive.CheckCommand
def CheckCommand
Definition: archive.py:90
archive.GetZippedFile
def GetZippedFile
Definition: archive.py:361
def archive.RemoveAndRegister (   newFile,
  oldFiles,
  logfile 
)

Definition at line 438 of file archive.py.

References Delete(), and filereg().

438 
439 def RemoveAndRegister(newFile,oldFiles, logfile):
440  for file in oldFiles.split():
441  newpath = newFile + "#" + file[len(dir):]
442  logfile.write("Registering New File Path " + newpath +"\n")
443  filereg(db,bakdb,tmpdb,newpath,logfile)
444  Delete(file, logfile)
445 
446 """
447 Remove and Unregister A File
448 Reference to fileunreg()
"""
archive.RemoveAndRegister
def RemoveAndRegister
Definition: archive.py:438
archive.Delete
def Delete
Definition: archive.py:449
archive.filereg
def filereg
Definition: archive.py:49
def archive.ScanDir (   file,
  logfile 
)

Definition at line 197 of file archive.py.

References CheckCommand(), ConfirmPath(), and split.

Referenced by CheckPath().

197 
198 def ScanDir(file, logfile) :
199  mycmd = "rfdir "
200  myarg = targetdir
201  cmd = mycmd + myarg
202  logfile.write("Scanning tape area " + cmd + "\n")
203  result = CheckCommand(cmd, logfile)
204  if result[0] == 0:
205  if cmp(result[1],"") != 0:
206  output = result[1].split('\n')
207  for line in output :
208  if len(line.split()) > 8:
209  newpath = targetdir + line.split()[-1]
210  logfile.write("Looking for File at " + newpath + "\n")
211  confirmpath = ConfirmPath(file, newpath, logfile)
212  logfile.write("Returned Path " + confirmpath + "\n")
213  if cmp(confirmpath, newpath + "/" + file[len(arcdir):] ) == 0: return confirmpath
214  return emptyString
215 
216 # path check over
217 #
218 
219 #
220 # T0 transfer functions
221 """
222 Transfer File with T0 System
223 Reference to CheckCommand()
"""
archive.CheckCommand
def CheckCommand
Definition: archive.py:90
archive.ConfirmPath
def ConfirmPath
Definition: archive.py:173
archive.ScanDir
def ScanDir
Definition: archive.py:197
split
double split
Definition: MVATrainer.cc:139
def archive.SetPath (   file,
  logfile 
)

Definition at line 141 of file archive.py.

References CheckPath(), and filereg().

Referenced by Cleaner().

141 
142 def SetPath(file, logfile):
143  path = CheckPath(file,logfile)
144  if cmp(path,emptyString) != 0:
145  newpath = "rfio:" + path
146  logfile.write("Register New Path " + newpath + "\n")
147  filereg(db,bakdb,tmpdb,newpath,logfile)
148  return True
149  else :logfile.write("File Transferred, but not found on tape\n")
150  return False
151 
152 """
153 Path Specifier
154 Reference to ConfirmPath(), ScanDir()
"""
archive.CheckPath
def CheckPath
Definition: archive.py:155
archive.SetPath
def SetPath
Definition: archive.py:141
archive.filereg
def filereg
Definition: archive.py:49
def archive.TransferWithT0System (   filepath,
  flag,
  logfile 
)

Definition at line 224 of file archive.py.

References CheckCommand(), and split.

Referenced by CheckFileStatus().

224 
225 def TransferWithT0System(filepath, flag, logfile):
226  filename = filepath[len(arcdir):]
227  nrun = filepath[len(arcdir)+len("DQM_Online_R"):-len("_R000064807.zip")]# file name length matters!
228  transfer_string = transferScript + " --runnumber " + nrun + " --path " + arcdir + " --filename " + filename
229  if EnableTransfer is False: transfer_string += " --test "# TEST, no file transfer
230  if flag is True: transfer_string += " --renotify "# transfer failed previously, trying to send it again
231  mycmd = transfer_string
232  myarg = fullTransferArg
233  cmd = mycmd + myarg
234  result = CheckCommand(cmd, logfile)
235  if result[0] == 0:
236  output = result[1].split('\n')
237  for line in output:
238  if line.find("File sucessfully submitted for transfer.") != -1 and flag is False:
239  logfile.write("File is queued " + filepath + "\n")
240  return True
241  if line.find("File sucessfully re-submitted for transfer.") != -1 and flag is True:
242  logfile.write("File is resubmitted " + filepath + "\n")
243  return True
244  if EnableTransfer is False: logfile.write(" *** Transfer Test Mode ***\n No File Transferred.\n") # TEST, no file transfer
245  return False
246 
247 """
248 Check File Status of Transferred File
249 Reference to CheckCommand(), TransferWithT0System()
"""
archive.TransferWithT0System
def TransferWithT0System
Definition: archive.py:224
archive.CheckCommand
def CheckCommand
Definition: archive.py:90
split
double split
Definition: MVATrainer.cc:139

Variable Documentation

string archive.arcdir = "/nfshome0/smaruyam/CMSSW_2_0_10/src/test/"

Definition at line 14 of file archive.py.

string archive.bakdb = dbdir+"tmp/backup.db"

Definition at line 38 of file archive.py.

string archive.cfgarg = " --config "

Definition at line 26 of file archive.py.

string archive.cfgfile = " /nfshome0/smaruyam/CMSSW_2_0_10/src/test/myconfig.txt "

Definition at line 15 of file archive.py.

string archive.db = dbdir+"db.db"

Definition at line 39 of file archive.py.

string archive.dbdir = "/nfshome0/smaruyam/CMSSW_2_0_10/src/test/"

Definition at line 13 of file archive.py.

string archive.dir = "/nfshome0/smaruyam/CMSSW_2_0_10/src/test/"

Definition at line 12 of file archive.py.

int archive.disk_threshold = 80

Definition at line 23 of file archive.py.

string archive.emptyString = "empty"

Definition at line 29 of file archive.py.

Referenced by edm::service::RandomNumberGeneratorService.fillDescriptions(), reco::modules::CosmicTrackSelector.produce(), reco::modules::AnalyticalTrackSelector.produce(), reco::modules::HICaloCompatibleTrackSelector.produce(), reco::modules::TrackMultiSelector.produce(), and FRDEventOutputModule< Consumer >.write().

archive.EnableFileRemoval = False

Definition at line 18 of file archive.py.

archive.EnableTransfer = False

Definition at line 20 of file archive.py.

archive.filepath = outputFileName

Definition at line 480 of file archive.py.

Referenced by HPDNoiseLibraryReader.HPDNoiseLibraryReader().

int archive.fileSizeThreshold = 1000000000

Definition at line 22 of file archive.py.

string archive.firstFile = "DQM_Online_"

Definition at line 470 of file archive.py.

Referenced by TauDQMFileLoader::cfgEntryFileSet.cfgEntryFileSet(), and fwlite::MultiChainEvent.MultiChainEvent().

archive.flag = False

Definition at line 489 of file archive.py.

Referenced by SiStripDcsInfo.addBadModules(), EcalSelectiveReadoutValidation.anaDigi(), SiStripBadComponentsDQMServiceReader.analyze(), EcalRecHitsValidation.analyze(), EcalSelectiveReadoutValidation.analyzeEB(), EcalSelectiveReadoutValidation.analyzeEE(), RPCDataCertification.beginRun(), RPCDCSSummary.beginRun(), RPCEventSummary.beginRun(), TangentApproachInRPhi.calculate(), DTTracoChip.calculateAngles(), SteppingAction.catchLongLived(), cond::service::DTHVCheckWithHysteresis.checkCurrentStatus(), cond::service::DTHVCheckByAbsoluteValues.checkCurrentStatus(), EcalUncalibratedRecHit.checkFlag(), EcalRecHit.checkFlag(), PFRecoTauAlgorithm.checkPos(), AlignmentTrackSelector.checkPrescaledHits(), EcalSelectiveReadoutValidation.checkSrApplication(), EgammaHLTNxNClusterProducer.checkStatusOfEcalRecHit(), HLTEcalResonanceFilter.checkStatusOfEcalRecHit(), StackingAction.ClassifyNewTrack(), ClosestApproachInRPhi.compute(), DTTime2DriftParametrization.computeDriftDistance_mean(), DTTime2DriftParametrization.computeDriftDistance_mode(), TShapeAnalysis.computeShape(), EcalDeadChannelRecoveryAlgos< DetIdT >.correct(), dccPhiIndexOfRU(), DDIsValid(), CSCValidation.doADCTiming(), CSCValidation.doGasGain(), DTDigitizer.driftTimeFromParametrization(), spr.eECALmatrix(), reco::PFCandidateElectronExtra.electronStatus(), reco::PFCandidateEGammaExtra.electronStatus(), SiStripBadStrip.encode(), EcalPerEvtLaserAnalyzer.endJob(), ZeeCalibration.endOfJob(), InvRingCalib.endOfLoop(), SiStripQualityChecker.fillDetectorStatus(), MELaserPrim.fillHistograms(), HLXMonitor.FillHistograms(), FWFileEntry.filterEventsWithCustomParser(), SiStripDaqInfo.findExcludedModule(), EcalSrFlag.flagName(), ConversionFinder.getConversionInfo(), SiStripTrackerMapCreator.getDetectorFlagAndComment(), SiStripQualityChecker.getModuleStatus(), RPCNoisyStripTest.getMonitorElements(), RPCBxTest.getMonitorElements(), RPCOccupancyTest.getMonitorElements(), RPCClusterSizeTest.getMonitorElements(), RPCDeadChannelTest.getMonitorElements(), RPCMultiplicityTest.getMonitorElements(), SiStripBadStripFromASCIIFile.getNewObject(), DTTracoLUTs.getPhiRad(), EcalSimRawData.getSrfs(), DreamSD.getStepInfo(), CaloSD.getStepInfo(), ECalSD.getTrackID(), HLTAlphaTFilter< T >.hltFilter(), ZMuMuAnalyzer_cynematics.isContained(), StackingAction.isItFromPrimary(), HCalSD.isItinFidVolume(), StackingAction.isItLongLived(), StackingAction.isItPrimaryDecayProductOrConversion(), reco::MuonSegmentMatch.isMask(), StackingAction.isThisRegion(), SteppingAction.killLowEnergy(), listbadmodule(), EcalUncalibRecHitRecWeightsAlgo< EBDataFrame >.makeRecHit(), EcalUncalibRecHitRecAnalFitAlgo< EBDataFrame >.makeRecHit(), L1MuGMTMerger.merge_rank(), reco::PFCandidateElectronExtra.mvaStatus(), reco::PFCandidateEGammaExtra.mvaStatus(), spr.newECALIdEW(), spr.newECALIdNS(), DDFilteredView.nextSibling(), graph< N, E >.nodeIndex(), DTHVStatusHandler.offlineList(), ZMuMuOverlap.operator()(), SiStripTrackerMapCreator.paintTkMapFromAlarm(), Generator.particlePassesPrimaryCuts(), CastorPedestalAnalysis.per2CapsHists(), HcalPedestalAnalysis.per2CapsHists(), pos::PixelFEDConfig.PixelFEDConfig(), HFFibreFiducial.PMTNumber(), SiStripActionExecutor.printFaultyModuleList(), EcalSelectiveReadoutProducer.printSrFlags(), EgammaHLTPixelMatchElectronAlgo.process(), VertexClassifier.processesAtSimulation(), TrackClassifier.processesAtSimulation(), HcalPedestalAnalysis.processEvent(), CastorPedestalAnalysis.processEvent(), EcalDetIdToBeRecoveredProducer.produce(), EcalDigiToRaw.produce(), read_badmodlist(), SiStripBadComponentsDQMService.readBadComponents(), JetPlusTrackCorrector.rebuildJta(), StackingAction.rrApplicable(), EcalSelectiveReadoutSuppressor.run(), ecaldqm::SelectiveReadoutTask.runOnDigis(), ecaldqm::SelectiveReadoutTask.runOnSrFlags(), KalmanAlignmentUserVariables.setAlignmentFlag(), reco::GsfElectron.setAmbiguous(), EcalFenixStripFgvbEE.setbadStripMissing(), EcalFenixStrip.setbadStripMissing(), objMonData< T >.setBJetsFlag(), objMon< T >.setBJetsFlag(), l1t::CaloCluster.setClusterFlag(), cond::DbConnectionConfiguration.setConnectionSharing(), L1MuDTTrackSegPhi.setEtaFlag(), lhef::JetInput.setExcludeResonances(), InputGenJetsParticleSelector.setExcludeResonances(), EcalRecHit.setFlag(), EcalUncalibratedRecHit.setFlagBit(), objMon< T >.setGenJets(), lhef::JetInput.setHardProcessOnly(), pat::Photon.setHasPixelSeed(), objMonData< T >.setL2MuFlag(), cond::persistency::ConnectionPool.setLogging(), reco::MuonSegmentMatch.setMask(), SiPixelFolderOrganizer.setModuleFolder(), lhef::JetInput.setPartonicFinalState(), InputGenJetsParticleSelector.setPartonicFinalState(), pat::Electron.setPassConversionVeto(), reco::GsfElectron.setPassCutBasedPreselection(), pat::Photon.setPassElectronVeto(), reco::GsfElectron.setPassMvaPreselection(), reco::GsfElectron.setPassPflowPreselection(), cond::DbConnectionConfiguration.setPoolAutomaticCleanUp(), cond::DbConnectionConfiguration.setReadOnlySessionOnUpdateConnections(), CaloTowersCreationAlgo.setRecoveredEcalHitsAreUsed(), CaloTowersCreationAlgo.setRecoveredHcalHitsAreUsed(), cond::DbConnectionConfiguration.setSQLMonitoring(), GflashHistogram.setStoreFlag(), lhef::JetInput.setTausAsJets(), InputGenJetsParticleSelector.setTausAsJets(), DatabasePDG.SetUseCharmParticles(), CaloTowersCreationAlgo.setUseRejectedHitsOnly(), CaloTowersCreationAlgo.setUseRejectedRecoveredEcalHits(), CaloTowersCreationAlgo.setUseRejectedRecoveredHcalHits(), sim::LocalFieldManager.SetVerbosity(), SortMuonSegmentMatches.SortMuonSegmentMatches(), MaterialBudgetForward.stopAfter(), MaterialBudget.stopAfter(), L1MuGMTLFMergeRankEtaQLUT.TheLookupFunction(), HistoryBase.traceSimHistory(), SimTrackManager.trackExists(), TangentApproachInRPhi.transverseCoord(), ClosestApproachInRPhi.transverseCoord(), EcalRecHit.unsetFlag(), L1TTestsSummary.updateL1TOccupancyMonitor(), L1TTestsSummary.updateL1TRateMonitor(), L1TTestsSummary.updateL1TSyncMonitor(), magfieldparam::BFit3D.UseSignedRad(), and magfieldparam::BFit3D.UseSpline().

string archive.fullTransferArg = cfgarg+" --type dqm --hostname srv-C2D05-19 --lumisection 1 --appname CMSSW --appversion CMSSW_2_0_10 "

Definition at line 27 of file archive.py.

tuple archive.lastFile = zipFileList.split()

Definition at line 471 of file archive.py.

Referenced by TauDQMFileLoader::cfgEntryFileSet.cfgEntryFileSet(), CmsShowNavigator.isLastEvent(), and fwlite::MultiChainEvent.MultiChainEvent().

tuple archive.logfile = open('archival_log.txt', 'a')

Definition at line 36 of file archive.py.

string archive.outputFileName = arcdir+firstFile+lastFile+".zip"

Definition at line 472 of file archive.py.

archive.PathReplace = False

Definition at line 19 of file archive.py.

string archive.sqlite3 = "sqlite3 "

Definition at line 32 of file archive.py.

string archive.statusCheck = cfgarg+" --check --filename "

Definition at line 28 of file archive.py.

string archive.targetdir = "/castor/cern.ch/cms/store/dqm/"

Definition at line 25 of file archive.py.

string archive.tmpdb = dbdir+"tmp/tmp.db"

Definition at line 37 of file archive.py.

tuple archive.transfer = TransferWithT0System(filepath,flag, logfile)

Definition at line 490 of file archive.py.

string archive.transferScript = "/nfshome0/tier0/scripts/injectFileIntoTransferSystem.pl"

Definition at line 24 of file archive.py.

tuple archive.zip = zipfile.ZipFile(outputFileName, "w")

Definition at line 476 of file archive.py.

Referenced by dirstructure.Directory.__create_pie_image(), argparse.ArgumentParser._parse_known_args(), python.rootplot.argparse.ArgumentParser._parse_known_args(), python.rootplot.root2matplotlib.HistStack.barstack(), ValidationMatrix_v2.ReleaseComparison.compare(), plotscripts.corrections2D(), geometryXMLparser.cscorder(), ValidationMatrix.do_comparisons_threaded(), geometryXMLparser.dtorder(), conddb_migrate.fetch_gts(), FileBlob.FileBlob(), ValidationMatrix.get_filenames_from_pool(), specificLumi.getFillFromDB(), specificLumi.getSpecificLumi(), ValidationMatrix.guess_params(), mpl_axes_hist_fix.hist(), dataformats.indent(), tablePrinter.indent(), PrescaleChecker.isMonotonic(), makeLayoutFileForGui.layDefaults(), MultipleCompare.main(), geometryDiff.matrixmult(), python.rootplot.utilities.Hist.min(), makeMuonMisalignmentScenario.mmult(), plotscripts.philines(), SteerMultipleCompare.plotOneByOne(), matplotRender.matplotRender.plotPeakPerday_Time(), matplotRender.matplotRender.plotPerdayX_Time(), matplotRender.matplotRender.plotSumX_Fill(), matplotRender.matplotRender.plotSumX_Run(), matplotRender.matplotRender.plotSumX_Time(), TablePrint.PrettyPrintTable(), TablePrint.PrintLine(), cmsPerfSuiteHarvest.process_igprof_dir(), cmsPerfSuiteHarvest.process_memcheck_dir(), cmsPerfSuiteHarvest.process_timesize_dir(), MatrixReader.MatrixReader.readMatrix(), python.rootplot.rootinfo.recurse_thru_file(), PixelFitterByConformalMappingAndLine.run(), TrackFitter.run(), PixelFitterByHelixProjections.run(), L1MuonPixelTrackFitter.run(), RecoTauValidation_cfi.SetYmodulesToLog(), ValidationUtils.SpawnPSet(), PrescaleChecker.TrendingWithLumi(), python.rootplot.utilities.wilson_interval(), Formatter.SimpleHTMLFormatter.writeStyledRow(), and geometryXMLparser.MuonGeometry.xml().

tuple archive.zipFileList = GetListOfFiles(logfile)

Definition at line 467 of file archive.py.

tuple archive.zipFileSize = os.path.getsize(filepath)

Definition at line 481 of file archive.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5