test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
fileCollector2 Namespace Reference

Functions

def convert
 
def filecheck
 
def isFileOpen
 
def logme
 
def processSiStrip
 
def uploadFile
 

Variables

list COLLECTING_DIR = sys.argv[1]
 
int COLLECTOR_WAIT_TIME = 10
 
list DROPBOX = sys.argv[3]
 
tuple EXEDIR = os.path.dirname(__file__)
 
string f = "%s/%s"
 
string finalTdir = "%s/%s/%s"
 
string finalTfile = "%s/%s"
 
tuple fList = sorted(glob("%s/*_%s_R%s*_d" % (finalTdir,subsystem, run)),cmp=lambda x,y: "_T" not in x and 1 or ("_T" in y and ( -1 * cmp(x,y))))
 
tuple fMatch = re.match('^(DQM|Playback)_V[0-9]{4}_(?P<subsys>.*)_R(?P<runnr>[0-9]{9})\.root$',f)
 
string fName = "%s/%s"
 
int KEEP = 2
 
tuple LAST_FILE_UPLOADED = time.time()
 
string LAST_SEEN_RUN = "0"
 
dictionary NEW = {}
 ENDLESS LOOP WITH SLEEP. More...
 
int RETRIES = 3
 
list run = NEW[rFile]
 
tuple runnr = fMatch.group("runnr")
 
string STOP_FILE = "%s/.stop"
 
tuple subsystem = fMatch.group("subsys")
 
list T_FILE_DONE_DIR = sys.argv[2]
 
tuple TAGS = sorted(glob('%s/tagfile_runend_*' % COLLECTING_DIR ),reverse=True)
 
tuple TMP_DROPBOX = os.path.join(DROPBOX,".uploading")
 
tuple tmpF = open(tmpFName,"w+")
 
string tmpFName = "%s/%s.root"
 
 transferred = False
 
int WAIT_TIME_FILE_PT = 60
 

Function Documentation

def fileCollector2.convert (   infile,
  ofile 
)

Definition at line 69 of file fileCollector2.py.

Referenced by processSiStrip().

69 
70 def convert(infile, ofile):
71  cmd = EXEDIR + '/convert.sh ' + infile + ' ' +ofile
72  os.system(cmd)
def fileCollector2.filecheck (   rootfile)

Definition at line 27 of file fileCollector2.py.

References logme(), SiPixelLorentzAngle_cfi.read, and split.

27 
28 def filecheck(rootfile):
29  cmd = EXEDIR + '/filechk.sh ' + rootfile
30  a = os.popen(cmd).read().split()
31  tag=a.pop()
32  if tag == '(int)(-1)':
33  logme("ERROR: File %s corrupted (isZombi)", rootfile)
34  return False
35  elif tag == '(int)0':
36  logme("ERROR: File %s is incomplete", rootfile)
37  return False
38  elif tag == '(int)1':
39  return True
40  else:
41  return False
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7
split
double split
Definition: MVATrainer.cc:139
def fileCollector2.isFileOpen (   fName)

Definition at line 42 of file fileCollector2.py.

42 
43 def isFileOpen(fName):
44  fName = os.path.realpath(fName)
45  pids=os.listdir('/proc')
46  for pid in sorted(pids):
47  try:
48  if not pid.isdigit():
49  continue
50 
51  if os.stat(os.path.join('/proc',pid)).st_uid != os.getuid():
52  continue
53 
54  uid = os.stat(os.path.join('/proc',pid)).st_uid
55  fd_dir=os.path.join('/proc', pid, 'fd')
56  if os.stat(fd_dir).st_uid != os.getuid():
57  continue
58 
59  for f in os.listdir(fd_dir):
60  fdName = os.path.join(fd_dir, f)
61  if os.path.islink(fdName) :
62  link=os.readlink(fdName)
63  if link == fName:
64  return True
65  except:
66  continue
67 
68  return False
def fileCollector2.logme (   msg,
  args 
)

Definition at line 23 of file fileCollector2.py.

Referenced by filecheck(), processSiStrip(), and uploadFile().

23 
24 def logme(msg, *args):
25  procid = "[%s/%d]" % (__file__.rsplit("/", 1)[-1], os.getpid())
26  print datetime.now(), procid, msg % args
def fileCollector2.processSiStrip (   fName,
  finalTfile 
)

Definition at line 121 of file fileCollector2.py.

References convert(), and logme().

121 
122 def processSiStrip(fName,finalTfile):
123  dqmfile = fName
124  if "Playback" in fName and "SiStrip" == NEW[rFile]["subSystem"]:
125  dqmfile = fName.replace('Playback','DQM')
126  convert(fName,dqmfile)
127  if not os.path.exists(dqmfile):
128  logme("ERROR: Problem converting %s skiping" % Tfile)
129  shutil.move(fName,finalTfile+"_d")
130  return (dqmfile,False)
131 
132  os.rename(fName,finalTfile.replace('Playback','Playback_full'))
133 
134  return (dqmfile,True)
def fileCollector2.uploadFile (   fName,
  subsystem,
  run 
)

Definition at line 73 of file fileCollector2.py.

References mergeVDriftHistosByStation.file, logme(), and SiPixelLorentzAngle_cfi.read.

73 
74 def uploadFile(fName, subsystem, run):
75  hname = os.getenv("HOSTNAME")
76  seed=hname.replace("-","t")[-6:]
77  finalTMPfile="%s/DQM_V0001_%s_R%s.root.%s" % (TMP_DROPBOX,subsystem,run,seed)
78  if os.path.exists(finalTMPfile):
79  os.remove(finalTMPfile)
80 
81  md5Digest=md5(file(fName).read())
82  originStr="md5:%s %d %s" % (md5Digest.hexdigest(),os.stat(fName).st_size,fName)
83  originTMPFile="%s.origin" % finalTMPfile
84  originFile=open(originTMPFile,"w")
85  originFile.write(originStr)
86  originFile.close()
87  shutil.copy(fName,finalTMPfile)
88  if not os.path.exists(finalTMPfile) or not os.stat(finalTMPfile).st_size == os.stat(fName).st_size:
89  return False
90 
91  version=1
92  lFile=open("%s/lock" % TMP_DROPBOX ,"a")
93  lockf(lFile,LOCK_EX)
94  for vdir,vsubdir,vfiles in os.walk(DROPBOX):
95  if 'DQM_V0001_%s_R%s.root' % (subsystem,run) not in vfiles:
96  continue
97 
98  version += 1
99 
100  if not os.path.exists("%s/%04d" % (DROPBOX,version)):
101  os.makedirs("%s/%04d" % (DROPBOX,version))
102  os.chmod("%s/%04d" % (DROPBOX,version),2775)
103 
104  finalfile="%s/%04d/DQM_V0001_%s_R%s.root" % (DROPBOX,version,subsystem,run)
105  originFileName="%s.origin" % finalfile
106  try:
107  os.rename(finalTMPfile,finalfile)
108  os.rename(originTMPFile,originFileName)
109  os.chmod(finalfile,stat.S_IREAD|stat.S_IRGRP|stat.S_IROTH| stat.S_IWRITE|stat.S_IWGRP|stat.S_IWOTH)
110  os.chmod(originFileName,stat.S_IREAD|stat.S_IRGRP|stat.S_IROTH| stat.S_IWRITE|stat.S_IWGRP|stat.S_IWOTH)
111  except:
112  lockf(lFile,LOCK_UN)
113  lFile.close()
114  logme("ERROR: File %s upload failed to the DROPBOX" % fName)
115  return False
116 
117  logme("INFO: File %s has been successfully sent to the DROPBOX" % fName)
118  lockf(lFile,LOCK_UN)
119  lFile.close()
120  return True
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7

Variable Documentation

list fileCollector2.COLLECTING_DIR = sys.argv[1]

Definition at line 9 of file fileCollector2.py.

int fileCollector2.COLLECTOR_WAIT_TIME = 10

Definition at line 14 of file fileCollector2.py.

list fileCollector2.DROPBOX = sys.argv[3]

Definition at line 11 of file fileCollector2.py.

tuple fileCollector2.EXEDIR = os.path.dirname(__file__)

Definition at line 13 of file fileCollector2.py.

string fileCollector2.f = "%s/%s"

Definition at line 159 of file fileCollector2.py.

string fileCollector2.finalTdir = "%s/%s/%s"

Definition at line 193 of file fileCollector2.py.

string fileCollector2.finalTfile = "%s/%s"

Definition at line 200 of file fileCollector2.py.

tuple fileCollector2.fList = sorted(glob("%s/*_%s_R%s*_d" % (finalTdir,subsystem, run)),cmp=lambda x,y: "_T" not in x and 1 or ("_T" in y and ( -1 * cmp(x,y))))

Definition at line 249 of file fileCollector2.py.

Referenced by FWTEventList.Add(), FWTEventList.Enter(), FWCompactVerticalLayout.GetDefaultSize(), and FWCompactVerticalLayout.Layout().

tuple fileCollector2.fMatch = re.match('^(DQM|Playback)_V[0-9]{4}_(?P<subsys>.*)_R(?P<runnr>[0-9]{9})\.root$',f)

Definition at line 155 of file fileCollector2.py.

string fileCollector2.fName = "%s/%s"

Definition at line 280 of file fileCollector2.py.

int fileCollector2.KEEP = 2

Definition at line 17 of file fileCollector2.py.

tuple fileCollector2.LAST_FILE_UPLOADED = time.time()

Definition at line 138 of file fileCollector2.py.

fileCollector2.LAST_SEEN_RUN = "0"

Definition at line 137 of file fileCollector2.py.

dictionary fileCollector2.NEW = {}

ENDLESS LOOP WITH SLEEP.

Definition at line 136 of file fileCollector2.py.

int fileCollector2.RETRIES = 3

Definition at line 18 of file fileCollector2.py.

list fileCollector2.run = NEW[rFile]

Definition at line 191 of file fileCollector2.py.

tuple fileCollector2.runnr = fMatch.group("runnr")

Definition at line 157 of file fileCollector2.py.

string fileCollector2.STOP_FILE = "%s/.stop"

Definition at line 19 of file fileCollector2.py.

tuple fileCollector2.subsystem = fMatch.group("subsys")

Definition at line 158 of file fileCollector2.py.

list fileCollector2.T_FILE_DONE_DIR = sys.argv[2]

Definition at line 10 of file fileCollector2.py.

tuple fileCollector2.TAGS = sorted(glob('%s/tagfile_runend_*' % COLLECTING_DIR ),reverse=True)

Definition at line 149 of file fileCollector2.py.

tuple fileCollector2.TMP_DROPBOX = os.path.join(DROPBOX,".uploading")

Definition at line 16 of file fileCollector2.py.

tuple fileCollector2.tmpF = open(tmpFName,"w+")

Definition at line 286 of file fileCollector2.py.

string fileCollector2.tmpFName = "%s/%s.root"

Definition at line 273 of file fileCollector2.py.

fileCollector2.transferred = False

Definition at line 190 of file fileCollector2.py.

int fileCollector2.WAIT_TIME_FILE_PT = 60

Definition at line 15 of file fileCollector2.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5