CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
personalPlayback.Playback Class Reference
Inheritance diagram for personalPlayback.Playback:
personalPlayback.Applet

Public Member Functions

def discover_files
 
def do_create_lumi
 
def do_exec
 
def do_init
 
- Public Member Functions inherited from personalPlayback.Applet
def __init__
 
def do_exec
 
def do_init
 
def log
 
def write
 

Public Attributes

 global_file
 
 input
 
 lumi_backlog
 
 lumi_backlog_size
 
 lumi_found
 
 lumi_order
 
 next_lumi_index
 
 output
 
 ramdisk
 
 run
 
- Public Attributes inherited from personalPlayback.Applet
 control_fp
 
 kwargs
 
 name
 
 opts
 

Static Public Attributes

tuple re_pattern = re.compile(r'run([0-9]+)_ls([0-9]+)_stream([A-Za-z0-9]+)_([A-Za-z0-9_-]+)\.jsn')
 

Additional Inherited Members

- Static Public Member Functions inherited from personalPlayback.Applet
def read
 

Detailed Description

Definition at line 64 of file personalPlayback.py.

Member Function Documentation

def personalPlayback.Playback.discover_files (   self)

Definition at line 67 of file personalPlayback.py.

67 
68  def discover_files(self):
69  self.lumi_found = {}
70 
71  files_found = set()
72  streams_found = set()
73  run_found = None
74  for f in os.listdir(self.input):
75  r = self.re_pattern.match(f)
76  if r:
77  run, lumi, stream, stream_source = r.groups()
78  run, lumi = int(run), int(lumi)
79 
80  if run_found is None:
81  run_found = run
82  elif run_found != run:
83  raise Exception("Files from multiple runs are not (yet) supported for as playback input.")
84 
85  lumi_dct = self.lumi_found.setdefault(lumi, { 'streams': {} })
86  lumi_dct["streams"][stream] = (f, stream_source)
87  files_found.add(f)
88  streams_found.add(stream)
89 
90  if run_found is None:
91  raise Exception("Playback files not found.")
92 
93  if self.run < 0:
94  self.run = run_found
95 
96  self.log.info("Found run %s, will map output to run %s", run_found, self.run)
97  self.log.info("Found %d lumisections with %d files", len(self.lumi_found), len(files_found))
98  self.log.info("Found %d streams: %s", len(streams_found), list(streams_found))
99 
100  self.lumi_order = list(self.lumi_found.keys())
101  self.lumi_order.sort()
102  self.log.info("Lumi order: %s", str(self.lumi_order))
#define str(s)
def personalPlayback.Playback.do_create_lumi (   self)

Definition at line 126 of file personalPlayback.py.

References WorkFlow.WorkFlow.input, personalPlayback.Playback.input, personalPlayback.Playback.lumi_backlog, personalPlayback.Playback.lumi_backlog_size, personalPlayback.Playback.lumi_found, personalPlayback.Playback.lumi_order, personalPlayback.Playback.next_lumi_index, GeometryComparisonPlotter.output, personalPlayback.Playback.output, PhysicsTools::Calibration::MVAComputer.output, RPCRunIOV::RunIOV_Item.run, RPCDQMObject.run, TB06Reco.run, TB06RecoH2.run, EfficiencyPlotter.run, WZInterestingEventSelector::event.run, MuonRecoTest.run, DTResolutionAnalysisTest.run, Point.run, BeamSpotRcdReader::theBSfromDB.run, DTDataIntegrityTest.run, BeamSpotOnlineRecordsReader::theBSOfromDB.run, DTBlockedROChannelsTest.run, DTResolutionTest.run, DTChamberEfficiencyTest.run, DTEfficiencyTest.run, ME::Header.run, personalPlayback.Playback.run, DQMNet::CoreObject.run, DTLocalTriggerBaseTest.run, MatrixUtil.InputInfo.run, HitEff.run, cond::RunInfo_t.run, and SiPixelErrorEstimation.run.

Referenced by personalPlayback.Playback.do_exec().

127  def do_create_lumi(self):
128  orig_lumi = self.lumi_order[(self.next_lumi_index - 1) % len(self.lumi_order)]
129  play_lumi = self.next_lumi_index;
130  self.next_lumi_index += 1
131 
132  self.log.info("Start copying lumi (original) %06d -> %06d (playback)", orig_lumi, play_lumi)
133 
134  lumi_dct = self.lumi_found[orig_lumi]
135  streams = lumi_dct["streams"]
136 
137  def ijoin(f):
138  return os.path.join(self.input, f)
139 
140  def ojoin(f):
141  return os.path.join(self.output, f)
142 
143  written_files = set()
144  for stream, v in streams.items():
145  jsn_orig_fn, stream_source = v
146  jsn_play_fn = "run%06d_ls%04d_stream%s_%s.jsn" % (self.run, play_lumi, stream, stream_source)
147 
148  # define dat filename
149  ext = "dat"
150  if stream.startswith("streamDQMHistograms"):
151  ext = "pb"
152  dat_play_fn = "run%06d_ls%04d_stream%s_%s.%s" % (self.run, play_lumi, stream, stream_source, ext)
153 
154  # read the original file name, for copying
155  with open(ijoin(jsn_orig_fn), 'r') as f:
156  jsn_data = json.load(f)
157  dat_orig_fn = jsn_data["data"][3]
158 
159  # copy the data file
160  if os.path.exists(ijoin(dat_orig_fn)):
161  self.log.info("C: %s -> %s", dat_orig_fn, dat_play_fn)
162  shutil.copyfile(ijoin(dat_orig_fn), ojoin(dat_play_fn))
163 
164  written_files.add(dat_play_fn)
165  else:
166  log.warning("Dat file is missing: %s", dat_orig_fn)
167 
168  # write a new json file point to a different data file
169  # this has to be atomic!
170  jsn_data["data"][3] = dat_play_fn
171 
172  f = tempfile.NamedTemporaryFile(prefix=jsn_play_fn+ ".", suffix=".tmp", dir = self.output, delete=False)
173  tmp_fp = f.name
174  json.dump(jsn_data, f)
175  f.close()
176 
177  os.rename(tmp_fp, ojoin(jsn_play_fn))
178  written_files.add(jsn_play_fn)
179 
180  self.log.info("Copied %d files for lumi %06d", len(written_files), play_lumi)
181 
182  self.lumi_backlog.append((play_lumi, written_files))
183  while len(self.lumi_backlog) > self.lumi_backlog_size:
184  old_lumi, files_to_delete = self.lumi_backlog.popleft()
185 
186  self.log.info("Deleting %d files for old lumi %06d", len(files_to_delete), old_lumi)
187  for f in files_to_delete:
188  os.unlink(ojoin(f))
def personalPlayback.Playback.do_exec (   self)

Definition at line 189 of file personalPlayback.py.

References personalPlayback.Playback.do_create_lumi(), GeometryComparisonPlotter.output, personalPlayback.Playback.output, PhysicsTools::Calibration::MVAComputer.output, RPCRunIOV::RunIOV_Item.run, RPCDQMObject.run, TB06Reco.run, TB06RecoH2.run, EfficiencyPlotter.run, WZInterestingEventSelector::event.run, MuonRecoTest.run, DTResolutionAnalysisTest.run, Point.run, BeamSpotRcdReader::theBSfromDB.run, DTDataIntegrityTest.run, BeamSpotOnlineRecordsReader::theBSOfromDB.run, DTBlockedROChannelsTest.run, DTChamberEfficiencyTest.run, DTResolutionTest.run, DTEfficiencyTest.run, ME::Header.run, personalPlayback.Playback.run, DQMNet::CoreObject.run, DTLocalTriggerBaseTest.run, MatrixUtil.InputInfo.run, HitEff.run, cond::RunInfo_t.run, and SiPixelErrorEstimation.run.

190  def do_exec(self):
191  last_write = 0
192  lumi_produced = 0
193 
194  while True:
195  time.sleep(1)
196 
197  now = time.time()
198  if (now - last_write) > self.opts.playback_time_lumi:
199  last_write = now
200 
201  if self.opts.playback_nlumi > -1 and lumi_produced >= self.opts.playback_nlumi:
202  break
203 
204  self.do_create_lumi()
205  lumi_produced += 1
206 
207  # write eor
208  eor_fn = "run%06d_ls0000_EoR.jsn" % (self.run, )
209  eor_fp = os.path.join(self.output, eor_fn)
210  with open(eor_fp, "w"):
211  pass
212 
213  self.log.info("Wrote EoR: %s", eor_fp)
def personalPlayback.Playback.do_init (   self)

Definition at line 103 of file personalPlayback.py.

104  def do_init(self):
105  # check if our input directory is okay
106  self.input = self.opts.playback
107  self.ramdisk = self.opts.work_ramdisk
108  self.run = self.opts.run
109  self.log.info("Using input directory: %s", self.input)
110 
111  self.discover_files()
113  self.output = os.path.join(self.ramdisk, "run%06d" % self.run)
114  if not os.path.isdir(self.output):
115  os.makedirs(self.output)
116  self.log.info("Using output directory: %s", self.output)
118  self.global_file = os.path.join(self.ramdisk, ".run%06d.global" % self.run)
119  self.log.info("Writing: %s", self.global_file)
120  with open(self.global_file, "w") as f:
121  f.write("run_key = pp_run")
123  self.lumi_backlog = collections.deque()
125  self.next_lumi_index = 1

Member Data Documentation

personalPlayback.Playback.global_file

Definition at line 117 of file personalPlayback.py.

personalPlayback.Playback.input

Definition at line 105 of file personalPlayback.py.

Referenced by personalPlayback.Playback.do_create_lumi().

personalPlayback.Playback.lumi_backlog

Definition at line 122 of file personalPlayback.py.

Referenced by personalPlayback.Playback.do_create_lumi().

personalPlayback.Playback.lumi_backlog_size

Definition at line 123 of file personalPlayback.py.

Referenced by personalPlayback.Playback.do_create_lumi().

personalPlayback.Playback.lumi_found

Definition at line 68 of file personalPlayback.py.

Referenced by personalPlayback.Playback.do_create_lumi().

personalPlayback.Playback.lumi_order

Definition at line 99 of file personalPlayback.py.

Referenced by personalPlayback.Playback.do_create_lumi().

personalPlayback.Playback.next_lumi_index

Definition at line 124 of file personalPlayback.py.

Referenced by personalPlayback.Playback.do_create_lumi().

personalPlayback.Playback.output

Definition at line 112 of file personalPlayback.py.

Referenced by personalPlayback.Playback.do_create_lumi(), and personalPlayback.Playback.do_exec().

personalPlayback.Playback.ramdisk

Definition at line 106 of file personalPlayback.py.

Referenced by personalPlayback.FrameworkJob.discover_latest().

tuple personalPlayback.Playback.re_pattern = re.compile(r'run([0-9]+)_ls([0-9]+)_stream([A-Za-z0-9]+)_([A-Za-z0-9_-]+)\.jsn')
static

Definition at line 65 of file personalPlayback.py.

personalPlayback.Playback.run

Definition at line 93 of file personalPlayback.py.

Referenced by generateEDF.LumiInfo.__str__(), DTWorkflow.DTWorkflow.all(), Types.EventID.cppID(), Types.LuminosityBlockID.cppID(), personalPlayback.Playback.do_create_lumi(), personalPlayback.Playback.do_exec(), o2olib.O2OTool.execute(), generateEDF.LumiInfo.fixXingInfo(), and dqm_interfaces.DirWalkerFile.walk().