mergeAndRegister.py

Go to the documentation of this file.

#! /usr/bin/env python

import os,time,sys,shutil,glob
from sets import Set
from datetime import datetime
import smtplib
from email.MIMEText import MIMEText
from ROOT import TFile

def sendmail(EmailAddress,run):
  s=smtplib.SMTP("localhost")
  tolist=[EmailAddress]
  body="File merge failed by unknown reason for run"+run
  msg = MIMEText(body)
  msg['Subject'] = "File merge failed."
  msg['From'] = ServerMail
  msg['To'] = EmailAddress
  s.sendmail(ServerMail,tolist,msg.as_string())
  s.quit()

def filecheck(rootfile):
  f = TFile(rootfile)
  if (f.IsZombie()):
      #print "File corrupted"
      f.Close()
      return 0
  else:
      hist = f.FindObjectAny("reportSummaryContents")
      #(skip filecheck for HcalTiming files!!)
      if (hist == None and rootfile.rfind('HcalTiming') == -1):
          #print "File is incomplete"
          f.Close()
          return 0
      else:
          #print "File is OK"
          f.Close()
          return 1
      


TempTag = TimeTag + '-tmp'
if not os.path.exists(TimeTag):
  os.system('touch -t 01010000 '+ TimeTag)


#### search new files
NFOUND = 0
NEW = {}
TAG = os.stat(TimeTag).st_mtime
for dir, subdirs, files in os.walk(DIR):
  paths = ["%s/%s" % (dir, x) for x in files]
  for f in [x for x in paths if re.search(r'/DQM_.*_R[0-9]{9}\.root$', x)]:
  if os.stat(f).st_mtime > TAG and f.index("DQM_Reference") < 0:
    NEW.get(f[-14:-5], []).append(f)
    NFOUND += 1

if not NFOUND:
  print 'Have not found new files...'
  os._exit(99)

os.system("ls -l %s" % TimeTag)
os.system('touch '+ TempTag)
print datetime.now()
print 'Found %d new file(s).' % NFOUND

#### loop for runs
newFiles = []
allOldFiles = []
for (run, files) in NEW.iteritems():
  runnr = "%09d" % long(run)
  destdir = "%s/%s/%s/%s" % (FILEDIR, runnr[0:3], runnr[3:6], runnr[6:9])
  oldfiles = []
  version = 1
  while True:
    destfile = "%s/DQM_V%04d_R%s.root" % (destdir, version, runnr)
    if not os.path.exists(destfile): break
    oldfiles.append(destfile)
    version += 1

  if not os.path.exists(destdir):
    os.makedirs(destdir)

  logfile = "%s.log" % destfile[:-4]
  tmpdestfile = "%s.tmp" % destfile

  print 'Run %s is being merged...' % run

  if os.path.exists(tmpdestfile):
    os.remove(tmpdestfile)

  ntries = 0
  while ntries < 30:
    LOGFILE = file(logfile, 'a')
    LOGFILE.write(os.popen('DQMMergeFile %s %s' % (tmpdestfile, " ".join(files))).read())
    if not os.path.exists(tmpdestfile):
      print 'Failed merging files for run %s. Try again after two minutes' % run
    time.sleep(WAITTIME)
    ntries += 1
    else:
    break

   os.rename(tmpdestfile, destfile)
   sendmail(YourEmail,run)
   allOldFiles.extend(oldfiles)
   newFiles.append(destfile)


if os.path.exists(TMPDB):
  os.remove(TMPDB)

if os.path.exists(DB):
  os.rename(DB, TMPDB)
else:
  logfile.write('*** INITIALISE DATABASE ***\n')
  logfile.write(os.popen('visDQMRegisterFile %s "/Global/Online/ALL" "Global run"' % TMPDB).read())

logfile.write('*** UNREGISTER %d OLD FILES ***\n' % len(allOldFiles))
while len(allOldFiles) > 0:
  (slice, rest) = (allOldFiles[0:50], allOldFiles[50:])
  logfile.write(os.popen('visDQMUnregisterFile %s %s' % (tmpdb, " ".join(slice))).read())
  allOldFiles = rest
  for old in slice:
    os.remove(old)

for file in newFiles:
  print 'Registering %s' % file
  logfile.write('*** REGISTER FILE %s ***\n' % file)
  logfile.write(os.popen('visDQMRegisterFile %s "/Global/Online/ALL" "Global run" %s' % (TMPDB, file)).read())
  print '%s registered' % file

os.rename(TMPDB, DB)
os.remove(TimeTag)
os.rename(TempTag, TimeTag)