CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
List of all members | Public Member Functions | Public Attributes
ztee.GZipLog Class Reference
Inheritance diagram for ztee.GZipLog:

Public Member Functions

def __init__
 
def finish
 
def flush_block
 
def handle_timeout
 
def try_flush
 
def write
 
def write_block
 

Public Attributes

 block
 
 block_size
 
 block_size_max
 
 block_timeout
 
 file
 
 file_max_size
 
 file_min_size
 
 file_truncate_pos
 
 file_truncate_state
 
 last_flush
 
 zstream
 

Detailed Description

Definition at line 13 of file ztee.py.

Constructor & Destructor Documentation

def ztee.GZipLog.__init__ (   self,
  log_file 
)

Definition at line 14 of file ztee.py.

14 
15  def __init__(self, log_file):
16  self.file = None
17 
18  self.last_flush = time.time()
19  self.zstream = zlib.compressobj(6, zlib.DEFLATED, zlib.MAX_WBITS | 16)
20 
21  # 'gzip' block will be flushed if we reach max bytes or the timeout
22  self.block = []
23  self.block_size = 0
24  self.block_size_max = 2*1024*1024 # bytes
25  self.block_timeout = 15 # seconds
26 
27  self.file = open(log_file, "wb+")
28  self.file_min_size = 2*1024*1024 # this amount of space we keep after truncating
29  self.file_max_size = 16*1024*1024 # we truncate if file gets bigger
30 
31  self.file_truncate_pos = None
32  self.file_truncate_state = None
file_min_size
Definition: ztee.py:27
last_flush
Definition: ztee.py:17
block_size
Definition: ztee.py:22
file_truncate_state
Definition: ztee.py:31
def __init__
Definition: ztee.py:14
block_timeout
Definition: ztee.py:24
file_max_size
Definition: ztee.py:28
file_truncate_pos
Definition: ztee.py:30
block_size_max
Definition: ztee.py:23

Member Function Documentation

def ztee.GZipLog.finish (   self)

Definition at line 70 of file ztee.py.

References fbuf.file, ztee.GZipLog.file, TkOfflineVariables.file, EcalRecHitsFilter.file, EcalBxOrbitNumberGrapher.file, SiStripPlotGain.file, SiStripCorrelateBadStripAndNoise.file, SiStripCorrelateNoise.file, DrawIteration::Input.file, EcalURecHitHists.file, GlobalTrackerMuonAlignment.file, EcalCosmicsHists.file, and ztee.GZipLog.flush_block().

Referenced by progressbar.ProgressBar.__next__().

70 
71  def finish(self):
72  self.flush_block()
73  self.file.write(self.zstream.flush(zlib.Z_FINISH))
74  self.file.close()
75  self.file = None
def finish
Definition: ztee.py:70
def flush_block
Definition: ztee.py:37
def ztee.GZipLog.flush_block (   self)

Definition at line 37 of file ztee.py.

References ztee.GZipLog.block, ztee.GZipLog.block_size, ztee.GZipLog.file_max_size, ztee.GZipLog.file_min_size, ztee.GZipLog.file_truncate_pos, ztee.GZipLog.file_truncate_state, join(), ztee.GZipLog.last_flush, ztee.GZipLog.write_block(), ztee.GZipLog.zstream, and ztail.Decoder.zstream.

Referenced by ztee.GZipLog.finish(), and ztee.GZipLog.try_flush().

37 
38  def flush_block(self):
39  self.last_flush = time.time()
40 
41  data, size, = "".join(self.block), self.block_size
42  self.block, self.block_size = [], 0
43 
44  if size == 0:
45  # nothing to do
46  return
47 
48  # check if we need to truncate the file
49  # 'size' refers to uncompressed data, so the file be truncated at max_size -+ max_block_size
50  # there is no way know compressed size without collapsing the zlib state
51  if (self.file.tell() + size) > self.file_max_size:
52  if self.file_truncate_pos is not None:
53  # tell it to cleanup
54  self.zstream.flush(zlib.Z_FINISH)
55 
56  self.file.seek(self.file_truncate_pos, 0)
57  self.file.truncate()
58  self.zstream = self.file_truncate_state.copy()
59 
60  self.write_block("\n\n--- file was cut at this point ---\n\n")
61 
62  # write it
63  self.write_block(data)
64 
65  # check if we can save the ptr into truncate_position
66  if self.file_truncate_pos is None:
67  if self.file.tell() >= self.file_min_size:
68  self.file_truncate_pos = self.file.tell()
69  self.file_truncate_state = self.zstream.copy()
file_min_size
Definition: ztee.py:27
last_flush
Definition: ztee.py:17
block_size
Definition: ztee.py:22
file_truncate_state
Definition: ztee.py:31
def flush_block
Definition: ztee.py:37
def write_block
Definition: ztee.py:33
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
file_max_size
Definition: ztee.py:28
file_truncate_pos
Definition: ztee.py:30
def ztee.GZipLog.handle_timeout (   self)

Definition at line 90 of file ztee.py.

References ztee.GZipLog.try_flush().

Referenced by esMonitoring.AsyncLineReaderTimeoutMixin.readable().

90 
91  def handle_timeout(self):
92  self.try_flush()
93 
def handle_timeout
Definition: ztee.py:90
def try_flush
Definition: ztee.py:76
def ztee.GZipLog.try_flush (   self)

Definition at line 76 of file ztee.py.

References ztee.GZipLog.block_size, ztee.GZipLog.block_size_max, ztee.GZipLog.block_timeout, ztee.GZipLog.flush_block(), and ztee.GZipLog.last_flush.

Referenced by ztee.GZipLog.handle_timeout(), and ztee.GZipLog.write().

76 
77  def try_flush(self):
78  timeout = (time.time() - self.last_flush) > self.block_timeout
79  large_block = self.block_size > self.block_size_max
80 
81  if timeout or large_block:
82  self.flush_block()
last_flush
Definition: ztee.py:17
block_size
Definition: ztee.py:22
def flush_block
Definition: ztee.py:37
block_timeout
Definition: ztee.py:24
block_size_max
Definition: ztee.py:23
def try_flush
Definition: ztee.py:76
def ztee.GZipLog.write (   self,
  bytes 
)

Definition at line 83 of file ztee.py.

References ztee.GZipLog.block_size, and ztee.GZipLog.try_flush().

Referenced by pkg.AbstractPkg.generate().

83 
84  def write(self, bytes):
85  if bytes:
86  self.block.append(bytes)
87  self.block_size += len(bytes)
88 
89  self.try_flush()
def write
Definition: ztee.py:83
block_size
Definition: ztee.py:22
def try_flush
Definition: ztee.py:76
def ztee.GZipLog.write_block (   self,
  data 
)

Definition at line 33 of file ztee.py.

Referenced by ztee.GZipLog.flush_block().

33 
34  def write_block(self, data):
35  self.file.write(self.zstream.compress( data.encode("utf-8") ))
36  self.file.write(self.zstream.flush(zlib.Z_FULL_FLUSH))
def write_block
Definition: ztee.py:33

Member Data Documentation

ztee.GZipLog.block

Definition at line 21 of file ztee.py.

Referenced by ztee.GZipLog.flush_block().

ztee.GZipLog.block_size

Definition at line 22 of file ztee.py.

Referenced by ztee.GZipLog.flush_block(), ztee.GZipLog.try_flush(), and ztee.GZipLog.write().

ztee.GZipLog.block_size_max

Definition at line 23 of file ztee.py.

Referenced by ztee.GZipLog.try_flush().

ztee.GZipLog.block_timeout

Definition at line 24 of file ztee.py.

Referenced by ztee.GZipLog.try_flush().

ztee.GZipLog.file

Definition at line 15 of file ztee.py.

Referenced by ztee.GZipLog.finish().

ztee.GZipLog.file_max_size

Definition at line 28 of file ztee.py.

Referenced by ztee.GZipLog.flush_block().

ztee.GZipLog.file_min_size

Definition at line 27 of file ztee.py.

Referenced by ztee.GZipLog.flush_block().

ztee.GZipLog.file_truncate_pos

Definition at line 30 of file ztee.py.

Referenced by ztee.GZipLog.flush_block().

ztee.GZipLog.file_truncate_state

Definition at line 31 of file ztee.py.

Referenced by ztee.GZipLog.flush_block().

ztee.GZipLog.last_flush

Definition at line 17 of file ztee.py.

Referenced by ztee.GZipLog.flush_block(), and ztee.GZipLog.try_flush().

ztee.GZipLog.zstream

Definition at line 18 of file ztee.py.

Referenced by ztee.GZipLog.flush_block().