fileCollector2 Namespace Reference

Functions
def	convert (infile, ofile)
def	filecheck (rootfile)
def	isFileOpen (fName)
def	logme (msg, args)
def	processSiStrip (fName, finalTfile)
def	uploadFile (fName, subsystem, run)

Variables
	cmp
	COLLECTING_DIR
	COLLECTOR_WAIT_TIME
	converted
	DROPBOX
	EXEDIR
	f
	finalTdir
	finalTfile
	fList
	fMatch
	fName
	fToUpload
	KEEP
	LAST_FILE_UPLOADED
	LAST_SEEN_RUN
	NEW
	ENDLESS LOOP WITH SLEEP. More...
	RETRIES
	reverse
	run
	runnr
	stdout
	STOP_FILE
	subsystem
	T_FILE_DONE_DIR
	TAGS
	TMP_DROPBOX
	tmpF
	tmpFName
	transferred
	WAIT_TIME_FILE_PT

Function Documentation

convert()

def fileCollector2.convert	(		infile,
			ofile
	)

Definition at line 71 of file fileCollector2.py.

def convert(infile, ofile):
  cmd = EXEDIR + '/convert.sh ' + infile + ' ' +ofile
  os.system(cmd)
  

filecheck()

def fileCollector2.filecheck

(

rootfile

)

Definition at line 29 of file fileCollector2.py.

References logme(), fileinputsource_cfi.read, and submitPVValidationJobs.split().

def filecheck(rootfile):
  cmd = EXEDIR + '/filechk.sh ' + rootfile
  a = os.popen(cmd).read().split()
  tag=a.pop()
  if tag == '(int)(-1)':
    logme("ERROR: File %s corrupted (isZombi)", rootfile)
    return False
  elif tag == '(int)0':
    logme("ERROR: File %s is incomplete", rootfile)
    return False
  elif tag == '(int)1':
    return True
  else:
    return False

isFileOpen()

def fileCollector2.isFileOpen

(

fName

)

Definition at line 44 of file fileCollector2.py.

def isFileOpen(fName):
  fName = os.path.realpath(fName)
  pids=os.listdir('/proc')
  for pid in sorted(pids):
    try:  
      if not pid.isdigit():
        continue
    
      if os.stat(os.path.join('/proc',pid)).st_uid != os.getuid():
        continue
      
      uid = os.stat(os.path.join('/proc',pid)).st_uid
      fd_dir=os.path.join('/proc', pid, 'fd')
      if os.stat(fd_dir).st_uid != os.getuid():
        continue
        
      for f in os.listdir(fd_dir):
        fdName = os.path.join(fd_dir, f)
        if os.path.islink(fdName) :       
            link=os.readlink(fdName)
            if link == fName:
              return True
    except:
      continue
          
  return False
     

logme()

def fileCollector2.logme	(		msg,
			args
	)

Definition at line 25 of file fileCollector2.py.

References print().

Referenced by filecheck(), processSiStrip(), and uploadFile().

def logme(msg, *args):
  procid = "[%s/%d]" % (__file__.rsplit("/", 1)[-1], os.getpid())
  print(datetime.now(), procid, msg % args)
  

processSiStrip()

def fileCollector2.processSiStrip	(		fName,
			finalTfile
	)

Definition at line 123 of file fileCollector2.py.

References logme().

def processSiStrip(fName,finalTfile):
  dqmfile = fName
  if "Playback" in fName and "SiStrip" == NEW[rFile]["subSystem"]:
    dqmfile = fName.replace('Playback','DQM')
    convert(fName,dqmfile)
    if not os.path.exists(dqmfile):
      logme("ERROR: Problem converting %s skiping" % Tfile)
      shutil.move(fName,finalTfile+"_d")
      return (dqmfile,False)
      
    os.rename(fName,finalTfile.replace('Playback','Playback_full'))
  
  return (dqmfile,True)
  

uploadFile()

def fileCollector2.uploadFile	(		fName,
			subsystem,
			run
	)

Definition at line 75 of file fileCollector2.py.

References geometryDiff.file, logme(), and fileinputsource_cfi.read.

def uploadFile(fName, subsystem, run):
  hname = os.getenv("HOSTNAME")
  seed=hname.replace("-","t")[-6:]
  finalTMPfile="%s/DQM_V0001_%s_R%s.root.%s" % (TMP_DROPBOX,subsystem,run,seed)        
  if os.path.exists(finalTMPfile):
    os.remove(finalTMPfile)
            
  md5Digest=md5(file(fName).read())
  originStr="md5:%s %d %s" % (md5Digest.hexdigest(),os.stat(fName).st_size,fName)
  originTMPFile="%s.origin" % finalTMPfile
  originFile=open(originTMPFile,"w")
  originFile.write(originStr)
  originFile.close() 
  shutil.copy(fName,finalTMPfile)
  if not os.path.exists(finalTMPfile) or not os.stat(finalTMPfile).st_size == os.stat(fName).st_size:
    return False
  
  version=1
  lFile=open("%s/lock" % TMP_DROPBOX ,"a")
  lockf(lFile,LOCK_EX)
  for vdir,vsubdir,vfiles in os.walk(DROPBOX):
    if 'DQM_V0001_%s_R%s.root' % (subsystem,run) not in vfiles:
      continue
    
    version += 1

  if not os.path.exists("%s/%04d" % (DROPBOX,version)):
    os.makedirs("%s/%04d" % (DROPBOX,version))
    os.chmod("%s/%04d" % (DROPBOX,version),2775)
                  
  finalfile="%s/%04d/DQM_V0001_%s_R%s.root" %   (DROPBOX,version,subsystem,run)        
  originFileName="%s.origin" % finalfile     
  try:
    os.rename(finalTMPfile,finalfile)
    os.rename(originTMPFile,originFileName)
    os.chmod(finalfile,stat.S_IREAD|stat.S_IRGRP|stat.S_IROTH| stat.S_IWRITE|stat.S_IWGRP|stat.S_IWOTH)
    os.chmod(originFileName,stat.S_IREAD|stat.S_IRGRP|stat.S_IROTH| stat.S_IWRITE|stat.S_IWGRP|stat.S_IWOTH)  
  except:
    lockf(lFile,LOCK_UN)
    lFile.close()
    logme("ERROR: File %s upload failed to the DROPBOX" % fName)
    return False
    
  logme("INFO: File %s has been successfully sent to the DROPBOX" % fName)
  lockf(lFile,LOCK_UN)
  lFile.close()
  return True
  

Variable Documentation

cmp

fileCollector2.cmp

Definition at line 251 of file fileCollector2.py.

COLLECTING_DIR

fileCollector2.COLLECTING_DIR

Definition at line 11 of file fileCollector2.py.

COLLECTOR_WAIT_TIME

fileCollector2.COLLECTOR_WAIT_TIME

Definition at line 16 of file fileCollector2.py.

converted

fileCollector2.converted

Definition at line 211 of file fileCollector2.py.

Referenced by MCPizeroAnalyzer.analyze(), edm.encode(), HistoManager.getDetIdsForType(), HistoManager.getElecIdsForType(), HcalDbService.makeHcalCalibration(), and HcalDbService.makeHcalCalibrationWidth().

DROPBOX

fileCollector2.DROPBOX

Definition at line 13 of file fileCollector2.py.

EXEDIR

fileCollector2.EXEDIR

Definition at line 15 of file fileCollector2.py.

f

fileCollector2.f

Definition at line 161 of file fileCollector2.py.

finalTdir

fileCollector2.finalTdir

Definition at line 195 of file fileCollector2.py.

finalTfile

fileCollector2.finalTfile

Definition at line 202 of file fileCollector2.py.

fList

fileCollector2.fList

Definition at line 251 of file fileCollector2.py.

Referenced by FWTEventList.Add(), FWTEventList.Enter(), FWCompactVerticalLayout.GetDefaultSize(), and FWCompactVerticalLayout.Layout().

fMatch

fileCollector2.fMatch

Definition at line 157 of file fileCollector2.py.

fName

fileCollector2.fName

Definition at line 282 of file fileCollector2.py.

Referenced by FilterSelection.acceptMap(), XMLDocument.addAttribute(), OMTFReconstruction.beginRun(), CaloMeanResponse.CaloMeanResponse(), classNameFinder(), PatternOptimizerBase.endJob(), OMTFPatternMaker.endJob(), OMTFReconstruction.endJob(), XMLConfigWriter.finaliseXMLDocument(), JetCorrector.getJetCorrector(), omtf::RpcLinkMap.init(), edm::RootInputFileSequence.initTheFile(), L1TMuonOverlapParamsESProducer.L1TMuonOverlapParamsESProducer(), L1TMuonOverlapPhase1ParamsESProducer.L1TMuonOverlapPhase1ParamsESProducer(), XMLDocument.newElement(), XMLDocument.newValue(), FilterSelection.print(), l1t::XmlConfigParser.readDOMFromFile(), CSCSectorReceiverLUT.readLUTsFromFile(), CaloMeanResponse.readResponse(), HCalSD.readWeightFromFile(), XMLConfigReader.setConfigFile(), XMLConfigReader.setEventsFile(), XMLConfigReader.setPatternsFile(), TTClusterAlgorithm< T >.TTClusterAlgorithm(), TTStubAlgorithm< T >.TTStubAlgorithm(), SummaryOutputProducer.write(), XMLConfigWriter.writeGPs(), EcalDccWeightBuilder.writeWeightToAsciiFile(), and EcalDccWeightBuilder.writeWeightToRootFile().

fToUpload

fileCollector2.fToUpload

Definition at line 211 of file fileCollector2.py.

KEEP

fileCollector2.KEEP

Definition at line 19 of file fileCollector2.py.

LAST_FILE_UPLOADED

fileCollector2.LAST_FILE_UPLOADED

Definition at line 140 of file fileCollector2.py.

LAST_SEEN_RUN

fileCollector2.LAST_SEEN_RUN

Definition at line 139 of file fileCollector2.py.

NEW

fileCollector2.NEW

ENDLESS LOOP WITH SLEEP.

Definition at line 138 of file fileCollector2.py.

RETRIES

fileCollector2.RETRIES

Definition at line 20 of file fileCollector2.py.

reverse

fileCollector2.reverse

Definition at line 151 of file fileCollector2.py.

run

fileCollector2.run

Definition at line 193 of file fileCollector2.py.

runnr

fileCollector2.runnr

Definition at line 159 of file fileCollector2.py.

stdout

fileCollector2.stdout

Definition at line 22 of file fileCollector2.py.

STOP_FILE

fileCollector2.STOP_FILE

Definition at line 21 of file fileCollector2.py.

subsystem

fileCollector2.subsystem

Definition at line 160 of file fileCollector2.py.

T_FILE_DONE_DIR

fileCollector2.T_FILE_DONE_DIR

Definition at line 12 of file fileCollector2.py.

TAGS

fileCollector2.TAGS

Definition at line 151 of file fileCollector2.py.

TMP_DROPBOX

fileCollector2.TMP_DROPBOX

Definition at line 18 of file fileCollector2.py.

tmpF

fileCollector2.tmpF

Definition at line 288 of file fileCollector2.py.

tmpFName

fileCollector2.tmpFName

Definition at line 275 of file fileCollector2.py.

transferred

fileCollector2.transferred

Definition at line 192 of file fileCollector2.py.

WAIT_TIME_FILE_PT

fileCollector2.WAIT_TIME_FILE_PT

Definition at line 17 of file fileCollector2.py.