Functions
def	forward_proxy (rundir)

def	write_HTCondor_submit_file (path, script, config, lib)

Variables
	action

	args = parser.parse_args(sys.argv[1:])

	backupCfgPath = os.path.join(Path, mergeCfg+".bak")

	backupScriptPath = os.path.join(Path, "theScript.sh.bak")

	cfgPath = os.path.join(Path, mergeCfg)

string	command = 'cat '

string	curJobName = 'm'

	default

	dest

bool	fire_htcondor = False

	help

	i = lib.nJobs

string	inCfgPath = theJobData+'/'

	int

bool	invalid_id = False

	job_id

	job_mask = lib.JOBDIR

	job_submit_file = write_HTCondor_submit_file(Path, scriptPath, mergeCfg, lib)

int	jobNumFrom1 = i+1

	lib = mpslib.jobdatabase()

	match = re.search('Job <(\d+)> is submitted', result)

	maxJobs

	mergeCfg = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)

bool	mergeOK = True

	nargs

	nMerge = i-lib.nJobs

int	nSub = 0

	parser

	Path = os.path.join(theJobData,lib.JOBDIR[i])

	resources = lib.get_class('mille')

	result

	scriptPath = os.path.join(Path, "theScript.sh")

string	submission = 'bsub -J %s %s %s/%s/theScript.sh'

	theJobData = os.path.join(os.getcwd(), "jobData")

string	theJobName = 'mpalign'

	type

	weight_conf = cPickle.load(f)

Function Documentation

def mps_fire.forward_proxy ( rundir )

Forward proxy to location visible from the batch system.

Arguments:
- `rundir`: directory for storing the forwarded proxy

Definition at line 26 of file mps_fire.py.

References digitizers_cfi.strip.

 def forward_proxy(rundir):
     """Forward proxy to location visible from the batch system.
 
     Arguments:
     - `rundir`: directory for storing the forwarded proxy
     """
 
     if not mps_tools.check_proxy():
         print "Please create proxy via 'voms-proxy-init -voms cms -rfc'."
         sys.exit(1)
 
     local_proxy = subprocess.check_output(["voms-proxy-info", "--path"]).strip()
     shutil.copyfile(local_proxy, os.path.join(rundir,".user_proxy"))
 
 

def mps_fire.write_HTCondor_submit_file	(	path,
		script,
		config,
		lib
	)

Writes 'job.submit' file in `path`.

Arguments:
- `path`: job directory
- `script`: script to be executed
- `config`: cfg file
- `lib`: MPS lib object

Definition at line 41 of file mps_fire.py.

References int, genParticles_cff.map, SiStripPI.max, split, and digitizers_cfi.strip.

 def write_HTCondor_submit_file(path, script, config, lib):
     """Writes 'job.submit' file in `path`.
 
     Arguments:
     - `path`: job directory
     - `script`: script to be executed
     - `config`: cfg file
     - `lib`: MPS lib object
     """
 
     resources = lib.get_class("pede").split("_")[1:] # strip off 'htcondor'
     job_flavour = resources[-1]
 
     job_submit_template="""\
 universe              = vanilla
 executable            = {script:s}
 output                = {jobm:s}/STDOUT
 error                 = {jobm:s}/STDOUT
 log                   = {jobm:s}/HTCJOB
 notification          = Always
 transfer_output_files = ""
 request_memory        = {pedeMem:d}M
 
 # adapted to space used on eos for binaries:
 request_disk          = {disk:d}
 
 # adapted to threads parameter in pede options and number of available cores
 request_cpus          = {cpus:d}
 
 +JobFlavour           = "{flavour:s}"
 """
     if "bigmem" in resources:
         job_submit_template += """\
 +BigMemJob            = True
 +AccountingGroup      = "group_u_CMS.e_cms_caf_bigmem"
 
 # automatically remove the job if the submitter has no permissions to run a BigMemJob
 periodic_remove       = !regexp("group_u_CMS.e_cms_caf_bigmem", AccountingGroup) && BigMemJob =?= True
 """
     job_submit_template += "\nqueue\n"
 
     print "Determine number of pede threads..."
     cms_process = mps_tools.get_process_object(os.path.join(Path, mergeCfg))
     pede_options = cms_process.AlignmentProducer.algoConfig.pedeSteerer.options.value()
     n_threads = 1
     for option in pede_options:
         if "threads" in option:
             n_threads = option.replace("threads", "").strip()
             n_threads = max(map(lambda x: int(x), n_threads.split()))
             break
     if n_threads > 16: n_threads = 16 # HTCondor machines have (currently) 16
                                       # cores, i.e. we ensure here that the job
                                       # would fit core-wise on one machine
 
     print "Determine required disk space on remote host..."
     # determine usage by each file instead of whole directory as this is what
     # matters for the specified disk usage:
     spco = subprocess.check_output # to make code below more less verbose
     opj = os.path.join             # dito
     cmd = ["du", "--apparent-size"]
     disk_usage = [int(item.split()[0])
                   for directory in ("binaries", "monitors", "tree_files")
                   for item
                   in spco(cmd+
                           glob.glob(opj(lib.mssDir, directory, "*"))).splitlines()]
     disk_usage = sum(disk_usage)
     disk_usage *= 1.1 # reserve 10% additional space
 
     job_submit_file = os.path.join(Path, "job.submit")
     with open(job_submit_file, "w") as f:
         f.write(job_submit_template.format(script = os.path.abspath(script),
                                            jobm = os.path.abspath(path),
                                            pedeMem = lib.pedeMem,
                                            disk = int(disk_usage),
                                            cpus = n_threads,
                                            flavour = job_flavour))
 
     return job_submit_file
 
 
 
 

Variable Documentation

mps_fire.action

Definition at line 132 of file mps_fire.py.

Referenced by FWPFCandidateDetailView.build(), FWTrackHitsDetailView.build(), FWConvTrackHitsDetailView.build(), CmsShowMainFrame.createNewViewerAction(), edm::EDLooperBase.doDuringLoop(), dqmd_manager.exec_func(), LHEXMLStringProduct.fillCompressedContent(), edm::StreamSchedule.finishedPaths(), cond::persistency::IOVEditor.flush(), FWGUIManager.FWGUIManager(), edm::Path.handleWorkerFailure(), edm::Worker.shouldRethrowException(), LHEXMLStringProduct.writeCompressedContent(), cond::persistency::ITagLogTable.~ITagLogTable(), and cond::persistency::TAG_LOG::Table.~Table().

mps_fire.args = parser.parse_args(sys.argv[1:])

Definition at line 146 of file mps_fire.py.

string mps_fire.backupCfgPath = os.path.join(Path, mergeCfg+".bak")

Definition at line 304 of file mps_fire.py.

mps_fire.backupScriptPath = os.path.join(Path, "theScript.sh.bak")

Definition at line 285 of file mps_fire.py.

string mps_fire.cfgPath = os.path.join(Path, mergeCfg)

Definition at line 305 of file mps_fire.py.

string mps_fire.command = 'cat '

Definition at line 296 of file mps_fire.py.

string mps_fire.curJobName = 'm'

Definition at line 355 of file mps_fire.py.

mps_fire.default

Definition at line 126 of file mps_fire.py.

Referenced by HcalTextCalibrations::CheckGetObject< T >.operator()().

mps_fire.dest

Definition at line 128 of file mps_fire.py.

Referenced by evf::EvFDaqDirector.checkTransferSystemPSet(), SimpleNavigableLayer.crossingState(), FieldHandler< TOutput, bool, char >.extractValue(), ASCIIBoolFieldHandler< TOutput, FalseCharacter >.extractValue(), FieldHandlerBase< TOutput >.getColumnName(), WatcherStreamFileReader.getInputFile(), npstat::ArrayND< Num1, Len1, Dim1 >.inPlaceMul(), EcalTrigPrimCompactColl.l1aSpike(), LzmaDec_DecodeReal(), LzmaDecode(), ecaldqm::MESetTrend.shift_(), CosmicMuonUtilities.stepPropagate(), and npstat.transposeBuffer().

bool mps_fire.fire_htcondor = False

Definition at line 192 of file mps_fire.py.

mps_fire.help

Definition at line 127 of file mps_fire.py.

mps_fire.i = lib.nJobs

Definition at line 269 of file mps_fire.py.

string mps_fire.inCfgPath = theJobData+'/'

Definition at line 322 of file mps_fire.py.

mps_fire.int

Definition at line 126 of file mps_fire.py.

Referenced by write_HTCondor_submit_file().

bool mps_fire.invalid_id = False

Definition at line 162 of file mps_fire.py.

mps_fire.job_id

Definition at line 155 of file mps_fire.py.

list mps_fire.job_mask = lib.JOBDIR

Definition at line 158 of file mps_fire.py.

mps_fire.job_submit_file = write_HTCondor_submit_file(Path, scriptPath, mergeCfg, lib)

Definition at line 301 of file mps_fire.py.

int mps_fire.jobNumFrom1 = i+1

Definition at line 271 of file mps_fire.py.

mps_fire.lib = mpslib.jobdatabase()

Definition at line 149 of file mps_fire.py.

mps_fire.match = re.search('Job <(\d+)> is submitted', result)

Definition at line 235 of file mps_fire.py.

mps_fire.maxJobs

Definition at line 154 of file mps_fire.py.

mps_fire.mergeCfg = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)

Definition at line 297 of file mps_fire.py.

bool mps_fire.mergeOK = True

Definition at line 261 of file mps_fire.py.

mps_fire.nargs

Definition at line 126 of file mps_fire.py.

mps_fire.nMerge = i-lib.nJobs

Definition at line 354 of file mps_fire.py.

int mps_fire.nSub = 0

Definition at line 212 of file mps_fire.py.

mps_fire.parser

Initial value:

 = argparse.ArgumentParser(
         description="Submit jobs that are setup in local mps database to batch system.",
 )

Definition at line 123 of file mps_fire.py.

mps_fire.Path = os.path.join(theJobData,lib.JOBDIR[i])

Definition at line 284 of file mps_fire.py.

Referenced by CSCEfficiency.CSCEfficiency(), and cms::DDNamespace.DDNamespace().

string mps_fire.resources = lib.get_class('mille')

Definition at line 197 of file mps_fire.py.

Referenced by MuonToSimAssociatorByHits.associateMuons(), MuonAssociatorByHits.associateRecoToSim(), MuonAssociatorByHits.associateSimToReco(), edm::PoolSource.PoolSource(), MuonToTrackingParticleAssociatorEDProducer.produce(), and edm::RootDelayedReader.RootDelayedReader().

string mps_fire.result

Initial value:

 = subprocess.check_output(submission,
                                                      stderr=subprocess.STDOUT,
                                                      shell=True)

Definition at line 226 of file mps_fire.py.

mps_fire.scriptPath = os.path.join(Path, "theScript.sh")

Definition at line 286 of file mps_fire.py.

list mps_fire.submission = 'bsub -J %s %s %s/%s/theScript.sh'

Definition at line 222 of file mps_fire.py.

mps_fire.theJobData = os.path.join(os.getcwd(), "jobData")

Definition at line 185 of file mps_fire.py.

mps_fire.theJobName = 'mpalign'

Definition at line 188 of file mps_fire.py.

mps_fire.type

Definition at line 126 of file mps_fire.py.

mps_fire.weight_conf = cPickle.load(f)

Definition at line 311 of file mps_fire.py.

Functions

Variables

Function Documentation

Variable Documentation