2 import os, time, sys, glob, re, shutil, stat, smtplib, socket
3 from email.MIMEText
import MIMEText
4 from fcntl
import lockf, LOCK_EX, LOCK_UN
5 from hashlib
import md5
6 from traceback
import print_exc, format_exc
7 from datetime
import datetime
8 from subprocess
import Popen,PIPE
9 sys.stdout = os.fdopen(sys.stdout.fileno(),
'w', 0)
12 COLLECTDIR = sys.argv[2]
13 TFILEDONEDIR = sys.argv[3]
18 EMAILINTERVAL = 15 * 60
19 SBASEDIR = os.path.abspath(__file__).rsplit(
"/",1)[0]
20 TMPDROPBOX =
"%s/.tmpdropbox" % DROPBOX
22 SENDMAIL =
"/usr/sbin/sendmail"
23 HOSTNAME = socket.gethostname().lower()
26 lastEmailSent = datetime.now()
30 procid =
"[%s/%d]" % (__file__.rsplit(
"/", 1)[-1], os.getpid())
31 print datetime.now(), procid, msg % args
34 cmd =
'root -l -b -q %s/filechk.C"(\\"%s\\")"' % (SBASEDIR,rootfile)
37 if tag ==
'(int)(-1)' or tag ==
'(int)0':
46 cmd =
'root -l -b -q %s/sistrip_reduce_file.C++"' \
47 '(\\"%s\\", \\"%s\\")" >& /dev/null' % (SBASEDIR,infile, ofile)
50 def sendmail(body="Hello from visDQMZipCastorVerifier"):
51 scall = Popen(
"%s -t" % SENDMAIL, shell=
True, stdin=PIPE)
52 scall.stdin.write(
"To: %s\n" % EMAIL)
53 scall.stdin.write(
"Subject: File Collector on server %s has a Critical Error\n" %
55 scall.stdin.write(
"\n")
56 scall.stdin.write(
"%s\n" % body)
60 logme(
"ERROR: Sendmail exit with status %s", rc)
63 if not os.path.exists(TMPDROPBOX):
64 os.makedirs(TMPDROPBOX)
66 if not os.path.exists(TFILEDONEDIR):
67 os.makedirs(TFILEDONEDIR)
69 if not os.path.exists(DROPBOX):
78 for dir, subdirs, files
in os.walk(COLLECTDIR):
80 fMatch=re.match(
'^DQM_V[0-9]{4}_(?P<subsys>.*)_R(?P<runnr>[0-9]{9})(|_T[0-9]*)\.root$',f)
82 fMatch=re.match(
'^Playback_V[0-9]{4}_(?P<subsys>.*)_R(?P<runnr>[0-9]{9})(|_T[0-9]*)\.root$', f)
85 runnr = int(fMatch.group(
"runnr"))
86 subsystem=fMatch.group(
"subsys")
88 donefile =
"%s/%s/%s/%s" % (TFILEDONEDIR, runstr[0:3], runstr[3:6], f)
89 f =
"%s/%s" % (dir, f)
90 if os.path.exists(donefile)
and os.stat(donefile).st_size == os.stat(f).st_size:
91 logme(
"WARNING: File %s was already processed but re-appeared", f)
95 NEW.setdefault(runnr, {}).setdefault(subsystem,[]).
append(f)
98 if len(NEW.keys()) == 0:
102 TAGS=sorted(glob.glob(
'%s/tagfile_runend_*' % COLLECTDIR ),reverse=
True)
104 if len(NEW.keys()) <= 1:
108 TAGRUNEND=long(sorted(NEW.keys(),reverse=
True)[1])
111 TAGRUNEND=long(TAGS[0].
split(
"_")[2])
116 for run,subsystems
in NEW.items():
120 for subsystem,files
in subsystems.items():
123 Tfiles=sorted(files,cmp=
lambda x,y:
"_T" not in x
and x != y
and 1
or cmp(x,y))[::-1]
125 seed=HOSTNAME.replace(
"-",
"t")[-6:]
126 finalTMPfile=
"%s/DQM_V0001_%s_R%09d.root.%s" % (TMPDROPBOX,subsystem,run,seed)
128 finalTfile=
"%s/%s/%s/%s" % (TFILEDONEDIR,runstr[0:3],runstr[3:6],Tfile.split(
"/")[-1])
129 finalTdir=
"%s/%s/%s" % (TFILEDONEDIR,runstr[0:3],runstr[3:6])
130 if not os.path.exists(finalTdir):
131 os.makedirs(finalTdir)
133 if os.path.exists(finalTMPfile):
134 os.remove(finalTMPfile)
139 shutil.move(Tfile,finalTfile+
"_d")
147 logme(
"INFO: File %s is incomplete looking for next"
148 " DQM_V*_%s_R%09d_T*.root valid file",
149 Tfile, subsystem, run)
152 shutil.move(Tfile,finalTfile+
"_d")
159 if "Playback" in Tfile
and "SiStrip" in Tfile:
160 dqmfile = Tfile.replace(
'Playback',
'DQM')
162 if not os.path.exists(dqmfile):
163 logme(
"WARNING: Problem converting %s skiping", Tfile)
164 shutil.move(Tfile,finalTfile+
"_d")
167 os.rename(Tfile,finalTfile.replace(
'Playback',
'Playback_full'))
170 for i
in range(RETRIES):
172 originStr=
"md5:%s %d %s" % (md5Digest.hexdigest(),os.stat(Tfile).st_size,Tfile)
173 originTMPFile=
"%s.origin" % finalTMPfile
174 originFile=open(originTMPFile,
"w")
175 originFile.write(originStr)
177 shutil.copy(Tfile,finalTMPfile)
179 lFile=open(
"%s/lock" % TMPDROPBOX ,
"a")
181 for vdir,vsubdir,vfiles
in os.walk(DROPBOX):
182 if 'DQM_V0001_%s_R%09d.root' % (subsystem,run)
not in vfiles:
186 if not os.path.exists(
"%s/V%04d" % (DROPBOX,version)):
187 os.makedirs(
"%s/V%04d" % (DROPBOX,version))
189 finalfile=
"%s/V%04d/DQM_V0001_%s_R%09d.root" % (DROPBOX,version,subsystem,run)
190 originFileName=
"%s.origin" % finalfile
191 if os.path.exists(finalTMPfile)
and os.stat(finalTMPfile).st_size == os.stat(Tfile).st_size:
192 os.rename(Tfile,finalTfile)
193 os.rename(finalTMPfile,finalfile)
194 os.rename(originTMPFile,originFileName)
195 os.chmod(finalfile,stat.S_IREAD|stat.S_IRGRP|stat.S_IROTH| stat.S_IWRITE|stat.S_IWGRP|stat.S_IWOTH)
196 os.chmod(originFileName,stat.S_IREAD|stat.S_IRGRP|stat.S_IROTH| stat.S_IWRITE|stat.S_IWGRP|stat.S_IWOTH)
197 logme(
"INFO: File %s has been successfully sent to the DROPBOX" , Tfile)
202 logme(
"ERROR: Problem transfering final file for run"
203 " %09d. Retrying in %d", run, WAITTIME)
206 if now - EMAILINTERVAL > lastEmailSent:
207 sendmail(
"ERROR: Problem transfering final file for run"
208 " %09d.\n Retrying in %d seconds" % (run, WAITTIME))
216 except KeyboardInterrupt, e:
220 logme(
'ERROR: %s', e)
222 if now - EMAILINTERVAL > lastEmailSent:
223 sendmail (
'ERROR: %s\n%s' % (e, format_exc()))