CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
fileCollector2 Namespace Reference

Functions

def convert
 
def filecheck
 
def isFileOpen
 
def logme
 
def processSiStrip
 
def uploadFile
 

Variables

list COLLECTING_DIR = sys.argv[1]
 
int COLLECTOR_WAIT_TIME = 10
 
list DROPBOX = sys.argv[3]
 
tuple EXEDIR = os.path.dirname(__file__)
 
string f = "%s/%s"
 
string finalTdir = "%s/%s/%s"
 
string finalTfile = "%s/%s"
 
tuple fList = sorted(glob("%s/*_%s_R%s*_d" % (finalTdir,subsystem, run)),cmp=lambda x,y: "_T" not in x and 1 or ("_T" in y and ( -1 * cmp(x,y))))
 
tuple fMatch = re.match('^(DQM|Playback)_V[0-9]{4}_(?P<subsys>.*)_R(?P<runnr>[0-9]{9})\.root$',f)
 
string fName = "%s/%s"
 
int KEEP = 2
 
tuple LAST_FILE_UPLOADED = time.time()
 
string LAST_SEEN_RUN = "0"
 
dictionary NEW = {}
 ENDLESS LOOP WITH SLEEP. More...
 
int RETRIES = 3
 
list run = NEW[rFile]
 
tuple runnr = fMatch.group("runnr")
 
string STOP_FILE = "%s/.stop"
 
tuple subsystem = fMatch.group("subsys")
 
list T_FILE_DONE_DIR = sys.argv[2]
 
tuple TAGS = sorted(glob('%s/tagfile_runend_*' % COLLECTING_DIR ),reverse=True)
 
tuple TMP_DROPBOX = os.path.join(DROPBOX,".uploading")
 
tuple tmpF = open(tmpFName,"w+")
 
string tmpFName = "%s/%s.root"
 
 transferred = False
 
int WAIT_TIME_FILE_PT = 60
 

Function Documentation

def fileCollector2.convert (   infile,
  ofile 
)

Definition at line 71 of file fileCollector2.py.

Referenced by processSiStrip().

71 
72 def convert(infile, ofile):
73  cmd = EXEDIR + '/convert.sh ' + infile + ' ' +ofile
74  os.system(cmd)
def fileCollector2.filecheck (   rootfile)

Definition at line 29 of file fileCollector2.py.

References logme(), SiPixelLorentzAngle_cfi.read, and submitPVValidationJobs.split().

29 
30 def filecheck(rootfile):
31  cmd = EXEDIR + '/filechk.sh ' + rootfile
32  a = os.popen(cmd).read().split()
33  tag=a.pop()
34  if tag == '(int)(-1)':
35  logme("ERROR: File %s corrupted (isZombi)", rootfile)
36  return False
37  elif tag == '(int)0':
38  logme("ERROR: File %s is incomplete", rootfile)
39  return False
40  elif tag == '(int)1':
41  return True
42  else:
43  return False
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7
def fileCollector2.isFileOpen (   fName)

Definition at line 44 of file fileCollector2.py.

44 
45 def isFileOpen(fName):
46  fName = os.path.realpath(fName)
47  pids=os.listdir('/proc')
48  for pid in sorted(pids):
49  try:
50  if not pid.isdigit():
51  continue
52 
53  if os.stat(os.path.join('/proc',pid)).st_uid != os.getuid():
54  continue
55 
56  uid = os.stat(os.path.join('/proc',pid)).st_uid
57  fd_dir=os.path.join('/proc', pid, 'fd')
58  if os.stat(fd_dir).st_uid != os.getuid():
59  continue
60 
61  for f in os.listdir(fd_dir):
62  fdName = os.path.join(fd_dir, f)
63  if os.path.islink(fdName) :
64  link=os.readlink(fdName)
65  if link == fName:
66  return True
67  except:
68  continue
69 
70  return False
def fileCollector2.logme (   msg,
  args 
)

Definition at line 25 of file fileCollector2.py.

References print().

Referenced by filecheck(), processSiStrip(), and uploadFile().

25 
26 def logme(msg, *args):
27  procid = "[%s/%d]" % (__file__.rsplit("/", 1)[-1], os.getpid())
28  print(datetime.now(), procid, msg % args)
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def fileCollector2.processSiStrip (   fName,
  finalTfile 
)

Definition at line 123 of file fileCollector2.py.

References convert(), and logme().

123 
124 def processSiStrip(fName,finalTfile):
125  dqmfile = fName
126  if "Playback" in fName and "SiStrip" == NEW[rFile]["subSystem"]:
127  dqmfile = fName.replace('Playback','DQM')
128  convert(fName,dqmfile)
129  if not os.path.exists(dqmfile):
130  logme("ERROR: Problem converting %s skiping" % Tfile)
131  shutil.move(fName,finalTfile+"_d")
132  return (dqmfile,False)
133 
134  os.rename(fName,finalTfile.replace('Playback','Playback_full'))
135 
136  return (dqmfile,True)
def fileCollector2.uploadFile (   fName,
  subsystem,
  run 
)

Definition at line 75 of file fileCollector2.py.

References mergeVDriftHistosByStation.file, logme(), and SiPixelLorentzAngle_cfi.read.

75 
76 def uploadFile(fName, subsystem, run):
77  hname = os.getenv("HOSTNAME")
78  seed=hname.replace("-","t")[-6:]
79  finalTMPfile="%s/DQM_V0001_%s_R%s.root.%s" % (TMP_DROPBOX,subsystem,run,seed)
80  if os.path.exists(finalTMPfile):
81  os.remove(finalTMPfile)
82 
83  md5Digest=md5(file(fName).read())
84  originStr="md5:%s %d %s" % (md5Digest.hexdigest(),os.stat(fName).st_size,fName)
85  originTMPFile="%s.origin" % finalTMPfile
86  originFile=open(originTMPFile,"w")
87  originFile.write(originStr)
88  originFile.close()
89  shutil.copy(fName,finalTMPfile)
90  if not os.path.exists(finalTMPfile) or not os.stat(finalTMPfile).st_size == os.stat(fName).st_size:
91  return False
92 
93  version=1
94  lFile=open("%s/lock" % TMP_DROPBOX ,"a")
95  lockf(lFile,LOCK_EX)
96  for vdir,vsubdir,vfiles in os.walk(DROPBOX):
97  if 'DQM_V0001_%s_R%s.root' % (subsystem,run) not in vfiles:
98  continue
99 
100  version += 1
101 
102  if not os.path.exists("%s/%04d" % (DROPBOX,version)):
103  os.makedirs("%s/%04d" % (DROPBOX,version))
104  os.chmod("%s/%04d" % (DROPBOX,version),2775)
105 
106  finalfile="%s/%04d/DQM_V0001_%s_R%s.root" % (DROPBOX,version,subsystem,run)
107  originFileName="%s.origin" % finalfile
108  try:
109  os.rename(finalTMPfile,finalfile)
110  os.rename(originTMPFile,originFileName)
111  os.chmod(finalfile,stat.S_IREAD|stat.S_IRGRP|stat.S_IROTH| stat.S_IWRITE|stat.S_IWGRP|stat.S_IWOTH)
112  os.chmod(originFileName,stat.S_IREAD|stat.S_IRGRP|stat.S_IROTH| stat.S_IWRITE|stat.S_IWGRP|stat.S_IWOTH)
113  except:
114  lockf(lFile,LOCK_UN)
115  lFile.close()
116  logme("ERROR: File %s upload failed to the DROPBOX" % fName)
117  return False
118 
119  logme("INFO: File %s has been successfully sent to the DROPBOX" % fName)
120  lockf(lFile,LOCK_UN)
121  lFile.close()
122  return True
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7

Variable Documentation

list fileCollector2.COLLECTING_DIR = sys.argv[1]

Definition at line 11 of file fileCollector2.py.

int fileCollector2.COLLECTOR_WAIT_TIME = 10

Definition at line 16 of file fileCollector2.py.

list fileCollector2.DROPBOX = sys.argv[3]

Definition at line 13 of file fileCollector2.py.

tuple fileCollector2.EXEDIR = os.path.dirname(__file__)

Definition at line 15 of file fileCollector2.py.

string fileCollector2.f = "%s/%s"

Definition at line 161 of file fileCollector2.py.

string fileCollector2.finalTdir = "%s/%s/%s"

Definition at line 195 of file fileCollector2.py.

string fileCollector2.finalTfile = "%s/%s"

Definition at line 202 of file fileCollector2.py.

tuple fileCollector2.fList = sorted(glob("%s/*_%s_R%s*_d" % (finalTdir,subsystem, run)),cmp=lambda x,y: "_T" not in x and 1 or ("_T" in y and ( -1 * cmp(x,y))))

Definition at line 251 of file fileCollector2.py.

Referenced by FWTEventList.Add(), FWTEventList.Enter(), FWCompactVerticalLayout.GetDefaultSize(), and FWCompactVerticalLayout.Layout().

tuple fileCollector2.fMatch = re.match('^(DQM|Playback)_V[0-9]{4}_(?P<subsys>.*)_R(?P<runnr>[0-9]{9})\.root$',f)

Definition at line 157 of file fileCollector2.py.

string fileCollector2.fName = "%s/%s"

Definition at line 282 of file fileCollector2.py.

int fileCollector2.KEEP = 2

Definition at line 19 of file fileCollector2.py.

tuple fileCollector2.LAST_FILE_UPLOADED = time.time()

Definition at line 140 of file fileCollector2.py.

fileCollector2.LAST_SEEN_RUN = "0"

Definition at line 139 of file fileCollector2.py.

dictionary fileCollector2.NEW = {}

ENDLESS LOOP WITH SLEEP.

Definition at line 138 of file fileCollector2.py.

int fileCollector2.RETRIES = 3

Definition at line 20 of file fileCollector2.py.

list fileCollector2.run = NEW[rFile]

Definition at line 193 of file fileCollector2.py.

tuple fileCollector2.runnr = fMatch.group("runnr")

Definition at line 159 of file fileCollector2.py.

string fileCollector2.STOP_FILE = "%s/.stop"

Definition at line 21 of file fileCollector2.py.

tuple fileCollector2.subsystem = fMatch.group("subsys")

Definition at line 160 of file fileCollector2.py.

list fileCollector2.T_FILE_DONE_DIR = sys.argv[2]

Definition at line 12 of file fileCollector2.py.

tuple fileCollector2.TAGS = sorted(glob('%s/tagfile_runend_*' % COLLECTING_DIR ),reverse=True)

Definition at line 151 of file fileCollector2.py.

tuple fileCollector2.TMP_DROPBOX = os.path.join(DROPBOX,".uploading")

Definition at line 18 of file fileCollector2.py.

tuple fileCollector2.tmpF = open(tmpFName,"w+")

Definition at line 288 of file fileCollector2.py.

string fileCollector2.tmpFName = "%s/%s.root"

Definition at line 275 of file fileCollector2.py.

fileCollector2.transferred = False

Definition at line 192 of file fileCollector2.py.

int fileCollector2.WAIT_TIME_FILE_PT = 60

Definition at line 17 of file fileCollector2.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5