CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
fff_deleter Namespace Reference

Classes

class  FileDeleter
 

Functions

def cleanup_threshold
 
def daemon
 
def daemonize
 
def diskusage
 
def diskusage_bytes
 
def iterate
 
def lock
 
def parse_file_name
 

Variables

tuple deleter
 
tuple flog_ch = logging.StreamHandler()
 
tuple formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 
tuple log = logging.getLogger(__name__)
 
 log_capture = None
 
tuple log_capture_ch = logging.StreamHandler(log_capture)
 
tuple re_files = re.compile(r"^run(?P<run>\d+)/run(?P<runf>\d+)_ls(?P<ls>\d+)_.+\.(dat|raw)+(\.deleted)*")
 
tuple sock = lock("fff_deleter")
 
dictionary thresholds
 
string top = "/fff/ramdisk"
 

Function Documentation

def fff_deleter.cleanup_threshold (   top,
  threshold,
  action,
  string 
)

Definition at line 54 of file fff_deleter.py.

References iterate(), and AlCaHLTBitMon_ParallelJobs.p.

Referenced by fff_deleter.FileDeleter.run().

54 
55 def cleanup_threshold(top, threshold, action, string):
56  st = os.statvfs(top)
57  total = st.f_blocks * st.f_frsize
58  used = total - (st.f_bavail * st.f_frsize)
59  threshold = used - float(total * threshold) / 100
60 
61  def p(x):
62  return float(x) * 100 / total
63 
64  log.info("Using %d (%.02f%%) of %d space, %d (%.02f%%) above %s threshold.",
65  used, p(used), total, threshold, p(threshold), string)
66 
67  if threshold > 0:
68  iterate(top, threshold, action)
69  log.info("Done cleaning up for %s threshold.", string)
70  else:
71  log.info("Threshold %s not reached, doing nothing.", string)
def cleanup_threshold
Definition: fff_deleter.py:54
def fff_deleter.daemon (   deleter,
  log_capture,
  delay_seconds = 30 
)

Definition at line 216 of file fff_deleter.py.

217 def daemon(deleter, log_capture, delay_seconds=30):
218  while True:
219  deleter.run()
220 
221  if log_capture:
222  log_out = log_capture.getvalue()
223  log_capture.truncate(0)
224  deleter.make_report(log_out)
225 
226  time.sleep(delay_seconds)
def fff_deleter.daemonize (   logfile,
  pidfile 
)

Definition at line 227 of file fff_deleter.py.

228 def daemonize(logfile, pidfile):
229  # do the double fork
230  pid = os.fork()
231  if pid != 0:
232  sys.exit(0)
233 
234  os.setsid()
235  sys.stdin.close()
236  sys.stdout.close()
237  sys.stderr.close()
238 
239  fl = open(logfile, "a")
240  sys.stdout = fl
241  sys.stderr = fl
242 
243  pid = os.fork()
244  if pid != 0:
245  sys.exit(0)
246 
247  if pidfile:
248  f = open(pidfile, "w")
249  f.write("%d\n" % os.getpid())
250  f.close()
def fff_deleter.diskusage (   top)

Definition at line 72 of file fff_deleter.py.

Referenced by fff_deleter.FileDeleter.run().

72 
73 def diskusage(top):
74  st = os.statvfs(top)
75  total = st.f_blocks * st.f_frsize
76  used = total - (st.f_bavail * st.f_frsize)
77  return float(used) * 100 / total
def fff_deleter.diskusage_bytes (   top)

Definition at line 78 of file fff_deleter.py.

Referenced by fff_deleter.FileDeleter.make_report().

78 
79 def diskusage_bytes(top):
80  st = os.statvfs(top)
81  total = st.f_blocks * st.f_frsize
82  free = st.f_bavail * st.f_frsize
83  used = total - free
84 
85  return used, free, total
def diskusage_bytes
Definition: fff_deleter.py:78
def fff_deleter.iterate (   top,
  stopSize,
  action 
)

Definition at line 25 of file fff_deleter.py.

References alignCSCRings.action, and parse_file_name().

Referenced by cleanup_threshold().

25 
26 def iterate(top, stopSize, action):
27  # entry format (path, size)
28  collected = []
29 
30  for root, dirs, files in os.walk(top, topdown=True):
31  for name in files:
32  fp = os.path.join(root, name)
33  rl = os.path.relpath(fp, top)
34 
35  sort_key = parse_file_name(rl)
36  if sort_key:
37  fsize = os.stat(fp).st_size
38  if fsize == 0:
39  continue
40 
41  sort_key = parse_file_name(rl)
42  collected.append((sort_key, fp, fsize, ))
43 
44  # for now just use simple sort
45  collected.sort(key=lambda x: x[0])
46 
47  # do the action
48  for sort_key, fp, fsize in collected:
49  if stopSize <= 0:
50  break
51 
52  action(fp)
53  stopSize = stopSize - fsize
def parse_file_name
Definition: fff_deleter.py:16
def fff_deleter.lock (   pname)

Definition at line 208 of file fff_deleter.py.

Referenced by evf::FastMonitoringService.accumulateFileSize(), cond::service::PoolDBOutputService.appendSinceTime(), cscdqm::Collection.book(), cscdqm::EventProcessor.calcEMUFractionHisto(), ecaldqm.checkElectronicsMap(), ecaldqm.checkGeometry(), ecaldqm.checkTopology(), ecaldqm.checkTrigTowerMap(), cond::service::PoolDBOutputService.closeIOV(), StorageAccount.counter(), cond::service::PoolDBOutputService.createNewIOV(), cond::XMLAuthenticationService::XMLAuthenticationService.credentials(), evf::FastMonitoringService.dowork(), CSCMonitorObject.Fill(), CSCMonitorObject.GetBinContent(), CSCMonitorObject.GetBinError(), CSCMonitorObject.GetEntries(), evf::FastMonitoringService.getEventsProcessedForLumi(), CSCMonitorObject.GetMaximumBin(), cond::service::PoolDBOutputService.initDB(), edm::JobReport.inputFileClosed(), cond::service::PoolDBOutputService.lookUpRecord(), BeamSpotWorkflow.main(), DQMNet.onPeerConnect(), DQMNet.onPeerData(), evf::FastMonitoringService.postBeginJob(), evf::FastMonitoringService.preGlobalBeginLumi(), evf::FastMonitoringService.preGlobalEndLumi(), evf::FastMonitoringService.preModuleBeginJob(), evf::FastMonitoringService.prePathEvent(), evf::FastMonitoringService.preStreamBeginLumi(), evf::FastMonitoringService.preStreamEndLumi(), cscdqm::Dispatcher.processStandby(), edm::JobReport.reportAnalysisFile(), edm::JobReport.reportError(), evf::FastMonitoringService.reportEventsThisLumiInSource(), edm::JobReport.reportFallbackAttempt(), edm::JobReport.reportRandomStateFile(), edm::JobReport.reportSkippedEvent(), edm::JobReport.reportSkippedFile(), CSCMonitorObject.SetAxisRange(), CSCMonitorObject.setAxisTitle(), CSCMonitorObject.SetBinContent(), CSCMonitorObject.SetBinError(), ecaldqm.setElectronicsMap(), CSCMonitorObject.SetEntries(), ecaldqm.setGeometry(), CSCMonitorObject.SetMaximum(), CSCMonitorObject.SetNormFactor(), ecaldqm.setTopology(), ecaldqm.setTrigTowerMap(), StorageAccount::Stamp.Stamp(), cscdqm::EventProcessor.standbyEfficiencyHistos(), edm::ZombieKillerService.startThread(), evf::FastMonitoringService.stoppedLookingForFile(), cond::service::PoolDBOutputService.tagInfo(), StorageAccount::Stamp.tick(), cscdqm::EventProcessor.updateEfficiencyHistos(), cscdqm::EventProcessorMutex.updateFractionAndEfficiencyHistos(), cscdqm::Dispatcher.updateFractionAndEfficiencyHistos(), cscdqm::EventProcessor.updateFractionHistos(), cond::service::PoolDBOutputService.writeOne(), and cscdqm::EventProcessor.writeShifterHistograms().

209 def lock(pname):
210  sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
211  try:
212  sock.bind('\0' + pname)
213  return sock
214  except socket.error:
215  return None
def fff_deleter.parse_file_name (   rl)

Definition at line 16 of file fff_deleter.py.

Referenced by iterate().

16 
17 def parse_file_name(rl):
18  m = re_files.match(rl)
19  if not m:
20  return None
21 
22  d = m.groupdict()
23  sort_key = (int(d["run"]), int(d["runf"]), int(d["ls"]), )
24  return sort_key
def parse_file_name
Definition: fff_deleter.py:16

Variable Documentation

tuple fff_deleter.deleter
Initial value:
2  top = top,
3  thresholds = thresholds,
4  # put "41XXXXXXXXX@mail2sms.cern.ch" to send the sms
5  email_to = [
6  "dmitrijus.bugelskis@cern.ch",
7  "atanas.batinkov@cern.ch",
8  "daniel.joseph.duggan@cern.ch",
9  ],
10  report_directory = "/tmp/dqm_monitoring/",
11  fake = False,
12  )

Definition at line 279 of file fff_deleter.py.

Referenced by QcdLowPtDQM.~QcdLowPtDQM().

tuple fff_deleter.flog_ch = logging.StreamHandler()

Definition at line 307 of file fff_deleter.py.

tuple fff_deleter.formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

Definition at line 293 of file fff_deleter.py.

Referenced by SiPixelCalibDigiProducer.checkPixel(), SiPixelOfflineCalibAnalysisBase.checkPixel(), SiPixelActionExecutor.dumpBarrelModIds(), SiPixelActionExecutor.dumpEndcapModIds(), SiPixelInformationExtractor.findNoisyPixels(), extend_argparse.FixedWidthFormatter(), SiPixelSCurveCalibrationAnalysis.makeThresholdSummary(), RPCPackingModule.produce(), SiPixelRawToDigi.produce(), and SiPixelDigiToRaw.produce().

tuple fff_deleter.log = logging.getLogger(__name__)

Definition at line 13 of file fff_deleter.py.

tuple fff_deleter.log_capture = None

Definition at line 296 of file fff_deleter.py.

tuple fff_deleter.log_capture_ch = logging.StreamHandler(log_capture)

Definition at line 298 of file fff_deleter.py.

tuple fff_deleter.re_files = re.compile(r"^run(?P<run>\d+)/run(?P<runf>\d+)_ls(?P<ls>\d+)_.+\.(dat|raw)+(\.deleted)*")

Definition at line 15 of file fff_deleter.py.

tuple fff_deleter.sock = lock("fff_deleter")

Definition at line 261 of file fff_deleter.py.

Referenced by edm::storage::StatisticsSenderService.filePreCloseEvent().

dictionary fff_deleter.thresholds
Initial value:
1 = {
2  'rename': 60,
3  'delete': 80,
4  'email': 90,
5  }

Definition at line 273 of file fff_deleter.py.

Referenced by InitialClusteringStepBase._algoName(), muonisolation::NominalEfficiencyThresholds.add(), Basic2DGenericPFlowClusterizer.Basic2DGenericPFlowClusterizer(), Basic2DGenericTopoClusterizer.buildTopoCluster(), muonisolation::Cuts.Cuts(), muonisolation::NominalEfficiencyThresholds.dump(), LocalMaximumSeedFinder.LocalMaximumSeedFinder(), L1JetEtScaleOnlineProd.newObject(), L1HtMissScaleOnlineProd.newObject(), L1HfRingEtScaleOnlineProd.newObject(), operator<<(), PFlow2DClusterizerWithTime.PFlow2DClusterizerWithTime(), ESZeroSuppressionProducer.produce(), SpikeAndDoubleSpikeCleaner.SpikeAndDoubleSpikeCleaner(), SiStripFedZeroSuppression.suppress(), and muonisolation::NominalEfficiencyThresholds.thresholdValueForEfficiency().

string fff_deleter.top = "/fff/ramdisk"

Definition at line 272 of file fff_deleter.py.

Referenced by EcalPreshowerGeometry.alignmentTransformIndexLocal(), VpspScanAlgorithm.analyse(), DTChamberEfficiencyTask.analyze(), TTbar_Kinematics.analyze(), L1RCTElectronIsolationCard.calcElectronCandidates(), HEPTopTagger.check_cos_theta(), TopDecaySubset.checkShowerModel(), TopDecaySubset.checkWBosons(), CmsShowCommonPopup.CmsShowCommonPopup(), TGeoFromDddService.createManager(), SummaryGeneratorReadoutView.fill(), SummaryGeneratorControlView.fill(), get_transform(), GEMSimpleModel.getSimHitBx(), HTLogicalMapEntry.HTLogicalMapEntry(), TtFullHadHypothesis.hypo(), FWCompactVerticalLayout.Layout(), KinematicTree.motherParticle(), FWLayoutBuilder.nextHints(), TtDecayChannelSelector.operator()(), TopProjectorDeltaROverlap< Top, Bottom >.operator()(), FWTGeoRecoGeometryESProducer.produce(), TGeoMgrFromDdd.produce(), MEEEGeom.quadrant(), edm::service::MessageLogger.unEstablishModule(), FWConfigurationManager.writeToFile(), and edm::helper::IndexRangeAssociation::FastFiller.~FastFiller().