personalPlayback.FrameworkJob Class Reference

Inheritance diagram for personalPlayback.FrameworkJob:

Public Member Functions
def	discover_latest (self)
def	do_exec (self)
def	do_init (self)
def	make_args (self, run)
def	start_run (self, current)
Public Member Functions inherited from personalPlayback.Applet
def	__init__ (self, name, opts, kwargs)
def	do_exec (self)
def	do_init (self)
def	log (self)
def	write (self, fp)

Public Attributes
	cfg_file
	cfg_fp
	cfg_link
	cmsenv_path
	current_state
	exec_file
	home_path
	log_path
	name
	output_path
	ramdisk
	ramdisk_fp
	run
	tag
Public Attributes inherited from personalPlayback.Applet
	control_fp
	kwargs
	name
	opts

Private Member Functions
def	_find_release (self)
def	_prepare_files (self)
def	_set_name (self)

Additional Inherited Members
Static Public Member Functions inherited from personalPlayback.Applet
def	read (fp)

Detailed Description

Definition at line 246 of file personalPlayback.py.

Member Function Documentation

_find_release()

def personalPlayback.FrameworkJob._find_release (   self )

private

Definition at line 259 of file personalPlayback.py.

References personalPlayback.FrameworkJob.cfg_file.

    def _find_release(self):
        fp = os.path.realpath(self.cfg_file)
        while len(fp) > 3:
            bn = os.path.basename(fp)
            fp = os.path.dirname(fp)

            if bn == "src":
                break

        if len(fp) <= 3:
            raise Exception("Could not find the cmssw release area.")

        self.cmsenv_path = fp
        self.log.info("cmsenv path: %s", self.cmsenv_path)

_prepare_files()

def personalPlayback.FrameworkJob._prepare_files (   self )

private

Definition at line 274 of file personalPlayback.py.

    def _prepare_files(self):
        self.home_path = os.path.join(self.opts.work_home, "%s_%s" % (self.name, hex(id(self))))
        self.home_path = os.path.realpath(self.home_path)
        os.makedirs(self.home_path)

        self.log_path = self.opts.work_logs
        self.log.info("logs path: %s", self.log_path)

        self.exec_file = os.path.join(self.home_path, "startDqmRun.sh")
        self.log.info("Creating: %s", self.exec_file)
        f = open(self.exec_file, "w")
        template = start_dqm_job
        body = template.format(log_path=self.log_path, cmsRun=self.opts.cmsRun)
        f.write(body)
        f.close()
        os.chmod(self.exec_file, 0o755)

        cmsset_globs = ["/afs/cern.ch/cms/cmsset_default.sh", "/home/dqm*local/base/cmsset_default.sh"]
        cmsset_target = None
        for t in cmsset_globs:
            files =  glob.glob(t)
            for f in files:
                cmsset_target = f
                break

        if cmsset_target is not None:
            base = os.path.join(self.home_path, "base")
            os.makedirs(base)

            cmsset_link = os.path.join(base, "cmsset_default.sh")
            self.log.info("Linking : %s -> %s", cmsset_link, cmsset_target)
            os.symlink(cmsset_target, cmsset_link)
        else:
            self.log.warning("Couldn't find cmsset_default.sh, source it yourself!")

        current_link = os.path.join(self.home_path, "current")
        target = os.path.relpath(self.cmsenv_path, self.home_path)
        self.log.info("Linking : %s -> %s", current_link, target)
        os.symlink(target, current_link)

        # check if current is outside the release directory
        # otherwise scram gets stuck forever
        cp = os.path.commonprefix([self.home_path, self.cmsenv_path])
        if self.cmsenv_path == cp:
            self.log.error("Working directory (incl. control directory), have to be outside the cmssw release. Otherwise scram fails due to recursive links.")
            raise Exception("Invalid home_path: %s" % self.home_path)

        output_link = os.path.join(self.home_path, "output")
        output_target = os.path.realpath(self.opts.work_output)
        target = os.path.relpath(output_target, self.home_path)
        self.log.info("Linking : %s -> %s", output_link, target)
        os.symlink(target, output_link)
        self.output_path = output_link

        cfg_link = os.path.join(self.home_path, os.path.basename(self.cfg_file))
        target = self.cfg_fp
        self.log.info("Linking : %s -> %s", cfg_link, target)
        os.symlink(target, cfg_link)
        self.cfg_link = cfg_link

_set_name()

def personalPlayback.FrameworkJob._set_name (   self )

private

Definition at line 247 of file personalPlayback.py.

References personalPlayback.FrameworkJob.cfg_file, and join().

    def _set_name(self):
        x = os.path.basename(self.cfg_file)
        x = re.sub(r'(.*)\.py', r'\1', x)
        x = re.sub(r'(.*)_cfg', r'\1', x)
        x = re.sub(r'(.*)-live', r'\1', x)
        x = re.sub(r'(.*)_sourceclient', r'\1', x)
        x = re.sub(r'(.*)_dqm', r'\1', x)

        x = "".join([c for c in x if c.isalnum()])
        self.tag = x
        self.name = "cmssw_%s" % x

discover_latest()

def personalPlayback.FrameworkJob.discover_latest

(

self

)

Definition at line 365 of file personalPlayback.py.

References personalPlayback.int, SiStripPI.max, personalPlayback.Applet.opts, personalPlayback.Playback.ramdisk, personalPlayback.FrameworkJob.ramdisk, and personalPlayback.RunDesc.

Referenced by personalPlayback.FrameworkJob.do_exec().

    def discover_latest(self):
        re_run = re.compile(r'run([0-9]+)')
        re_global = re.compile(r'\.run([0-9]+)\.global')

        # find runs
        runs = {}
        globals = {}
        for x in os.listdir(self.ramdisk):
            m = re_run.match(x)
            if m:
                runs[int(m.group(1))] = x

            m = re_global.match(x)
            if m:
                globals[int(m.group(1))] = x

        # find max global for which there is a run directory
        run_set = set(runs.keys())
        run_set = run_set.intersection(globals.keys())

        if self.opts.run < 0:
            largest = max(run_set)
        else:
            largest = self.opts.run

        #self.log.info("Largest: %s", largest)
        global_fp = os.path.join(self.ramdisk, globals[largest])
        with open(global_fp, "r") as f:
            global_param = f.read()

        return RunDesc(
            run = largest,
            run_fp = os.path.join(self.ramdisk, runs[largest]),
            global_fp = global_fp,
            global_param = global_param,
        )

do_exec()

def personalPlayback.FrameworkJob.do_exec

(

self

)

Definition at line 415 of file personalPlayback.py.

References personalPlayback.FrameworkJob.current_state, personalPlayback.FrameworkJob.discover_latest(), info(), personalPlayback.Applet.log(), conddbCopyTest.CopyTest.log(), conditionUploadTest.UploadTest.log(), cond::persistency::Logger.log(), cond::CredentialStore.log(), crabFunctions.CrabTask.log, and personalPlayback.FrameworkJob.start_run().

    def do_exec(self):
        time.sleep(1)

        self.current_state = None

        while True:
            latest = self.discover_latest()
            if self.current_state is None or latest != self.current_state.desc:
                self.log.info("Found latest run: %s", latest)

                self.start_run(latest)

            if not self.current_state:
                self.log.info("Run not found, waiting 1 sec.")
            else:
                r = self.current_state.proc.poll()
                if r is not None:
                    self.log.info("Process exitted: %s", r)

                    return 0

            time.sleep(1)

do_init()

def personalPlayback.FrameworkJob.do_init

(

self

)

Definition at line 335 of file personalPlayback.py.

    def do_init(self):
        # check if our input directory is okay
        self.ramdisk = self.opts.work_ramdisk
        self.run = self.opts.run
        self.cfg_file = self.kwargs["cfg_file"]

        if not os.path.isfile(self.cfg_file):
            raise Exception("Configuration file not found: %s" % self.cfg_file)

        self.cfg_fp = os.path.realpath(self.cfg_file)
        self.ramdisk_fp = os.path.realpath(self.ramdisk)

        self._set_name()
        self._find_release()
        self._prepare_files()

make_args()

def personalPlayback.FrameworkJob.make_args	(		self,
			run
	)

Definition at line 351 of file personalPlayback.py.

References personalPlayback.FrameworkJob.cfg_link, personalPlayback.FrameworkJob.exec_file, personalPlayback.FrameworkJob.home_path, personalPlayback.FrameworkJob.output_path, personalPlayback.FrameworkJob.ramdisk_fp, and personalPlayback.str.

Referenced by personalPlayback.FrameworkJob.start_run().

    def make_args(self, run):
        args = []
        args.append("bash")                 # arg 0
        args.append(self.exec_file)         # arg 0
        args.append(self.home_path)         # home path
        args.append("slc6_amd64_gcc491")    # release
        args.append(self.output_path)       # cwd/output path
        args.append(str(run))               # run
        args.append(self.ramdisk_fp)        # ramdisk
        args.append(self.cfg_link)          # cmsRun arg 1
        args.append("runkey=pp_run")        # cmsRun arg 2

        return args

start_run()

def personalPlayback.FrameworkJob.start_run	(		self,
			current
	)

Definition at line 402 of file personalPlayback.py.

References personalPlayback.FrameworkJob.current_state, info(), join(), personalPlayback.Applet.log(), conddbCopyTest.CopyTest.log(), conditionUploadTest.UploadTest.log(), cond::persistency::Logger.log(), cond::CredentialStore.log(), crabFunctions.CrabTask.log, and personalPlayback.FrameworkJob.make_args().

Referenced by personalPlayback.FrameworkJob.do_exec().

    def start_run(self, current):
        old_state = self.current_state

        # kill the old run
        # nope, since it involves eof and i am lazy
        if old_state:
            return

        args = self.make_args(current.run)
        self.log.info("Executing: %s", " ".join(args))
        proc = subprocess.Popen(args, preexec_fn=preexec_kill_on_pdeath)
        self.current_state = RunState(desc=current, proc=proc)

Member Data Documentation

cfg_file

personalPlayback.FrameworkJob.cfg_file

Definition at line 339 of file personalPlayback.py.

Referenced by personalPlayback.FrameworkJob._find_release(), and personalPlayback.FrameworkJob._set_name().

cfg_fp

personalPlayback.FrameworkJob.cfg_fp

Definition at line 344 of file personalPlayback.py.

cfg_link

personalPlayback.FrameworkJob.cfg_link

Definition at line 332 of file personalPlayback.py.

Referenced by personalPlayback.FrameworkJob.make_args().

cmsenv_path

personalPlayback.FrameworkJob.cmsenv_path

Definition at line 271 of file personalPlayback.py.

current_state

personalPlayback.FrameworkJob.current_state

Definition at line 413 of file personalPlayback.py.

Referenced by personalPlayback.FrameworkJob.do_exec(), and personalPlayback.FrameworkJob.start_run().

exec_file

personalPlayback.FrameworkJob.exec_file

Definition at line 282 of file personalPlayback.py.

Referenced by personalPlayback.FrameworkJob.make_args().

home_path

personalPlayback.FrameworkJob.home_path

Definition at line 275 of file personalPlayback.py.

Referenced by personalPlayback.FrameworkJob.make_args().

log_path

personalPlayback.FrameworkJob.log_path

Definition at line 279 of file personalPlayback.py.

name

personalPlayback.FrameworkJob.name

Definition at line 257 of file personalPlayback.py.

Referenced by ElectronMVAID.ElectronMVAID.__call__(), FWLite.ElectronMVAID.__call__(), dirstructure.Directory.__create_pie_image(), DisplayManager.DisplayManager.__del__(), dqm_interfaces.DirID.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dirstructure.Comparison.__make_image(), core.autovars.NTupleVariable.__repr__(), core.autovars.NTupleObjectType.__repr__(), core.autovars.NTupleObject.__repr__(), core.autovars.NTupleCollection.__repr__(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.Service.__setattr__(), config.CFG.__str__(), counter.Counter.__str__(), average.Average.__str__(), FWLite.WorkingPoints._reformat_cut_definitions(), core.autovars.NTupleObjectType.addSubObjects(), core.autovars.NTupleObjectType.addVariables(), core.autovars.NTupleObjectType.allVars(), dataset.CMSDataset.buildListOfFiles(), dataset.LocalDataset.buildListOfFiles(), dataset.CMSDataset.buildListOfFilesDBS(), dirstructure.Directory.calcStats(), validation.Sample.digest(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), DisplayManager.DisplayManager.Draw(), TreeCrawler.Package.dump(), core.autovars.NTupleVariable.fillBranch(), core.autovars.NTupleObject.fillBranches(), core.autovars.NTupleCollection.fillBranchesScalar(), core.autovars.NTupleCollection.fillBranchesVector(), core.autovars.NTupleCollection.get_cpp_declaration(), core.autovars.NTupleCollection.get_cpp_wrapper_class(), core.autovars.NTupleCollection.get_py_wrapper_class(), utils.StatisticalTest.get_status(), production_tasks.Task.getname(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), VIDSelectorBase.VIDSelectorBase.initialize(), core.autovars.NTupleVariable.makeBranch(), core.autovars.NTupleObject.makeBranches(), core.autovars.NTupleCollection.makeBranchesScalar(), core.autovars.NTupleCollection.makeBranchesVector(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), production_tasks.MonitorJobs.run(), python.rootplot.utilities.Hist.TGraph(), python.rootplot.utilities.Hist.TH1F(), counter.Counter.write(), and average.Average.write().

output_path

personalPlayback.FrameworkJob.output_path

Definition at line 326 of file personalPlayback.py.

Referenced by personalPlayback.FrameworkJob.make_args().

ramdisk

personalPlayback.FrameworkJob.ramdisk

Definition at line 337 of file personalPlayback.py.

Referenced by personalPlayback.FrameworkJob.discover_latest().

ramdisk_fp

personalPlayback.FrameworkJob.ramdisk_fp

Definition at line 345 of file personalPlayback.py.

Referenced by personalPlayback.FrameworkJob.make_args().

run

personalPlayback.FrameworkJob.run

Definition at line 338 of file personalPlayback.py.

Referenced by generateEDF.LumiInfo.__str__(), DTWorkflow.DTWorkflow.all(), Types.EventID.cppID(), Types.LuminosityBlockID.cppID(), o2olib.O2OTool.execute(), generateEDF.LumiInfo.fixXingInfo(), and dqm_interfaces.DirWalkerFile.walk().

tag

personalPlayback.FrameworkJob.tag

Definition at line 256 of file personalPlayback.py.

Referenced by FWLite.ElectronMVAID.__call__(), Inspector.Inspector.Loop(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), and DTWorkflow.DTWorkflow.remote_out_path().