Inheritance diagram for personalPlayback.Playback:

Public Member Functions
def	discover_files (self)

def	do_create_lumi (self)

def	do_exec (self)

def	do_init (self)

Public Member Functions inherited from personalPlayback.Applet
def	__init__ (self, name, opts, kwargs)

def	do_exec (self)

def	do_init (self)

def	log (self)

def	write (self, fp)

Public Attributes
	global_file

	input

	lumi_backlog

	lumi_backlog_size

	lumi_found

	lumi_order

	next_lumi_index

	output

	ramdisk

	run

Public Attributes inherited from personalPlayback.Applet
	control_fp

	kwargs

	name

	opts

Additional Inherited Members
Static Public Member Functions inherited from personalPlayback.Applet
def	read (fp)

Detailed Description

Definition at line 63 of file personalPlayback.py.

Member Function Documentation

def personalPlayback.Playback.discover_files ( self )

Definition at line 66 of file personalPlayback.py.

     def discover_files(self):
         self.lumi_found = {}
 
         files_found = set()
         streams_found = set()
         run_found = None
         for f in os.listdir(self.input):
             r = self.re_pattern.match(f)
             if r:
                 run, lumi, stream, stream_source = r.groups()
                 run, lumi = int(run), int(lumi)
 
                 if run_found is None:
                     run_found = run
                 elif run_found != run:
                     raise Exception("Files from multiple runs are not (yet) supported for as playback input.")
 
                 lumi_dct = self.lumi_found.setdefault(lumi, { 'streams': {} })
                 lumi_dct["streams"][stream] = (f, stream_source)
                 files_found.add(f)
                 streams_found.add(stream)
 
         if run_found is None:
             raise Exception("Playback files not found.")
 
         if self.run < 0:
             self.run = run_found
 
         self.log.info("Found run %s, will map output to run %s", run_found, self.run)
         self.log.info("Found %d lumisections with %d files", len(self.lumi_found), len(files_found))
         self.log.info("Found %d streams: %s", len(streams_found), list(streams_found))
 
         self.lumi_order = list(self.lumi_found.keys())
         self.lumi_order.sort()
         self.log.info("Lumi order: %s", str(self.lumi_order))
 

def personalPlayback.Playback.do_create_lumi ( self )

Definition at line 125 of file personalPlayback.py.

Referenced by personalPlayback.Playback.do_exec().

     def do_create_lumi(self):
         orig_lumi = self.lumi_order[(self.next_lumi_index - 1) % len(self.lumi_order)]
         play_lumi = self.next_lumi_index;
         self.next_lumi_index += 1
 
         self.log.info("Start copying lumi (original) %06d -> %06d (playback)", orig_lumi, play_lumi)
 
         lumi_dct = self.lumi_found[orig_lumi]
         streams = lumi_dct["streams"]
 
         def ijoin(f):
             return os.path.join(self.input, f)
 
         def ojoin(f):
             return os.path.join(self.output, f)
 
         written_files = set()
         for stream, v  in streams.items():
             jsn_orig_fn, stream_source = v
             jsn_play_fn = "run%06d_ls%04d_stream%s_%s.jsn" % (self.run, play_lumi, stream, stream_source)
 
             # define dat filename
             ext = "dat"
             if stream.startswith("streamDQMHistograms"):
                 ext = "pb"
             dat_play_fn = "run%06d_ls%04d_stream%s_%s.%s" % (self.run, play_lumi, stream, stream_source, ext)
 
             # read the original file name, for copying
             with open(ijoin(jsn_orig_fn), 'r') as f:
                 jsn_data = json.load(f)
                 dat_orig_fn = jsn_data["data"][3]
 
             # copy the data file
             if os.path.exists(ijoin(dat_orig_fn)):
                 self.log.info("C: %s -> %s", dat_orig_fn, dat_play_fn)
                 shutil.copyfile(ijoin(dat_orig_fn), ojoin(dat_play_fn))
 
                 written_files.add(dat_play_fn)
             else:
                 log.warning("Dat file is missing: %s", dat_orig_fn)
 
             # write a new json file point to a different data file
             # this has to be atomic!
             jsn_data["data"][3] = dat_play_fn
 
             f = tempfile.NamedTemporaryFile(prefix=jsn_play_fn+ ".", suffix=".tmp", dir = self.output, delete=False)
             tmp_fp = f.name
             json.dump(jsn_data, f)
             f.close()
 
             os.rename(tmp_fp, ojoin(jsn_play_fn))
             written_files.add(jsn_play_fn)
 
         self.log.info("Copied %d files for lumi %06d", len(written_files), play_lumi)
 
         self.lumi_backlog.append((play_lumi, written_files))
         while len(self.lumi_backlog) > self.lumi_backlog_size:
             old_lumi, files_to_delete = self.lumi_backlog.popleft()
 
             self.log.info("Deleting %d files for old lumi %06d", len(files_to_delete), old_lumi)
             for f in files_to_delete:
                 os.unlink(ojoin(f))
 

def personalPlayback.Playback.do_exec ( self )

Definition at line 188 of file personalPlayback.py.

References personalPlayback.Playback.do_create_lumi(), GeometryComparisonPlotter.output, personalPlayback.Playback.output, PhysicsTools::Calibration::MVAComputer.output, RPCRunIOV::RunIOV_Item.run, TB06RecoH2.run, TB06Reco.run, RPCDQMObject.run, EfficiencyPlotter.run, WZInterestingEventSelector::event.run, MuonRecoTest.run, DTResolutionAnalysisTest.run, BeamSpotRcdReader::theBSfromDB.run, DTBlockedROChannelsTest.run, DTDataIntegrityTest.run, DTChamberEfficiencyTest.run, DTResolutionTest.run, ME::Header.run, DTEfficiencyTest.run, personalPlayback.Playback.run, DQMNet::CoreObject.run, MatrixUtil.InputInfo.run, DTLocalTriggerBaseTest.run, HitEff.run, cond::RunInfo_t.run, and SiPixelErrorEstimation.run.

     def do_exec(self):
         last_write = 0
         lumi_produced = 0
 
         while True:
             time.sleep(1)
 
             now = time.time()
             if (now - last_write) > self.opts.playback_time_lumi:
                 last_write = now
 
                 if self.opts.playback_nlumi > -1 and lumi_produced >= self.opts.playback_nlumi:
                     break
 
                 self.do_create_lumi()
                 lumi_produced += 1
 
         # write eor
         eor_fn = "run%06d_ls0000_EoR.jsn" % (self.run, )
         eor_fp = os.path.join(self.output, eor_fn)
         with open(eor_fp, "w"):
             pass
 
         self.log.info("Wrote EoR: %s", eor_fp)
 

def personalPlayback.Playback.do_init ( self )

Definition at line 102 of file personalPlayback.py.

     def do_init(self):
         # check if our input directory is okay
         self.input = self.opts.playback
         self.ramdisk = self.opts.work_ramdisk
         self.run = self.opts.run
         self.log.info("Using input directory: %s", self.input)
 
         self.discover_files()
 
         self.output = os.path.join(self.ramdisk, "run%06d" % self.run)
         if not os.path.isdir(self.output):
             os.makedirs(self.output)
         self.log.info("Using output directory: %s", self.output)
 
         self.global_file = os.path.join(self.ramdisk, ".run%06d.global" % self.run)
         self.log.info("Writing: %s", self.global_file)
         with open(self.global_file, "w") as f:
             f.write("run_key = pp_run")
 
         self.lumi_backlog = collections.deque()
         self.lumi_backlog_size = 10
         self.next_lumi_index = 1
 