CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
fileTransfer.py
Go to the documentation of this file.
1 #!/usr/bin/env python3
2 
3 from __future__ import print_function
4 import os, time, sys, shutil, glob, smtplib, re, subprocess
5 import getopt as gop
6 import zipfile as zp
7 from datetime import datetime
8 from email.MIMEText import MIMEText
9 
10 SOURCEDIR = "/dqmdata/dqm/merged" #Directory where the original files are located.
11 INJECTIONDIR = "/dqmdata/dqm/Tier0Shipping/inject" #Directory where files get placed once they have been sent.
12 TRASHCAN = "/dqmdata/dqm/trashcan"
13 HOSTNAME = "srv-C2D05-19"
14 CONFIGFILE = "/nfshome0/dqm/.transfer/myconfig.txt"
15 INJECTIONSCRIPT = "/nfshome0/tier0/scripts/injectFileIntoTransferSystem.pl"
16 
17 EMPTY_TRASHCAN = False
18 WAIT_TIME = 600
19 
21  newfiles={}
22  trashfiles=[]
23  filelist=[]
24  for dir1, subdirs, files in os.walk(SOURCEDIR):
25  for f in files:
26  try:
27  version=int(f.split("_V")[-1][:4])
28  vlfn=f.split("_V")[0]+f.split("_V")[1][4:]
29  except:
30  version=1
31  vlfn=f
32  if vlfn in newfiles.keys() and version >= newfiles[vlfn][0]:
33  newfiles[vlfn]=(version,"%s/%s" % (dir1,f))
34  elif vlfn not in newfiles.keys():
35  newfiles[vlfn]=(version,"%s/%s" % (dir1,f))
36  else:
37  trashfiles.append("%s/%s" % (dir1,f))
38 
39  return (newfiles,trashfiles)
40 #=====================================================================================
41 def injectFile(f,renotify=False):
42  fname=f.rsplit("/",1)[-1]
43  dname=f.rsplit("/",1)[0]
44  run=f.split("_R")[-1][:9]
45  iname="%s/%s" % (INJECTIONDIR,fname)
46  shutil.move(f,iname)
47  parameters=["--filename %s" % fname,
48  "--type dqm",
49  "--path %s" % INJECTIONDIR,
50  "--destination dqm",
51  "--hostname %s" % HOSTNAME,
52  "--config %s" % CONFIGFILE,
53  "--runnumber %s" % run,
54  "--lumisection 95",
55  "--numevents 834474816",
56  "--appname dqmArchive",
57  "--appversion dqmArchive_1_0"]
58  cmd="%s %s" % (INJECTIONSCRIPT," ".join(parameters))
59  result = subprocess.getstatusoutput(cmd)
60  if result[0] >= 1:
61  output = result[1]
62  print("Error injecting file %s to transfer system checking if it exists" % f)
63  chkparameters=["--check","--filename %s" % fname,"--config %s" % CONFIGFILE]
64  cmd="%s %s" % (INJECTIONSCRIPT," ".join(chkparameters))
65  result = subprocess.getstatusoutput(cmd)
66  if result[0]==1:
67  if "File not found in database" in result[1]:
68  print("Error: file %s not found in transfer database, check configuration" % f)
69  return 0
70  else:
71  print("Warning: file %s already exists in transfer database" % f)
72  return 2
73  else:
74  print("Error: problem checking database entry for file %s\n Error:%s" % (f,result[1]))
75  return 0
76  else:
77  print("File %s injected successfully" % f)
78  return 1
79 #=====================================================================================
81  while True:
82  #look for NEW files in SOURCEDIR and files that need to be cleared.
83  newfiles,trashfiles=getFileLists()
84 
85  #Making sure destination directories exist
86  if not os.path.exists(TRASHCAN):
87  os.makedirs(TRASHCAN)
88  if not os.path.exists(INJECTIONDIR):
89  os.makedirs(INJECTIONDIR)
90 
91  #Dealing with trash can
92  for tf in trashfiles:
93  if EMPTY_TRASHCAN:
94  os.remove(tf)
95  else:
96  tfname="%s/%s" % (TRASHCAN,tf.rsplit("/",1)[-1])
97  shutil.move(tf,tfname)
98 
99  #Down to bussines
100  for ver,f in newfiles.values():
101  ifr=injectFile(f)
102  time.sleep(WAIT_TIMEt )
103 #=====================================================================================
104 if __name__ == "__main__":
105  transferFiles()
def transferFiles()
Definition: fileTransfer.py:80
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def getFileLists()
Definition: fileTransfer.py:20
def injectFile(f, renotify=False)
Definition: fileTransfer.py:41