Functions
def	as_dict (config)

def	batchScriptCERN (theCMSSW_BASE, runindex, eosdir, lumiToRun, key, config)

def	check_proxy ()

def	forward_proxy (rundir)

def	get_status_output (args, *kwargs)

def	getFilesForRun (blob)

def	getLuminosity (homedir, minRun, maxRun, isRunBased, verbose)

def	isInJSON (run, jsonfile)

def	main ()

def	mkdir_eos (out_path)
	method to create recursively directories on EOS More...

def	write_HTCondor_submit_file (path, name, nruns, proxy_path=None)

Variables
	__author__

	__copyright__

	__credits__

	__email__

	__license__

	__maintainer__

	__version__

	action

	args

	bashdir = os.path.join(cwd,"BASH")

string	cmd2 = ' dasgoclient -limit=0 -query \'file run='+blob[0]+' dataset='+blob[1]+'\''

	config = ConfigParser.ConfigParser()
	parse the configuration file More...

	CopyRights

	count = pool.map(getFilesForRun,mytuple)

	cwd = os.getcwd()

	default

string	desc = """This is a description of %prog."""

	dest

dictionary	dictionary = {}

	eosdir = os.path.join("/store/group/alca_trackeralign",USER,"test_out",opts.taskname)

	err

	file_info = dict(zip(runs, count))

	files = file_info[run]

	help

	HOME = os.environ.get('HOME')

	input_CMSSW_BASE = os.environ.get('CMSSW_BASE')

def	inputDict = as_dict(config)

	job_submit_file = os.path.join(path, "job_"+name+".submit")

string	job_submit_template

	jsonDATA = json.load(myJSON)

	key = key.split(":", 1)[1]
	prepare the HTCondor submission files and eventually submit them More...

string	listOfFiles = '['

	local_proxy = subprocess.check_output(["voms-proxy-info", "--path"]).strip()

def	lumimask = inputDict["Input"]["lumimask"]

	lumiToCache = line.split(",")[-1].replace("\r", "")

dictionary	myCachedLumi = {}

def	myLumiDB = getLuminosity(HOME,runs[0],runs[-1],True,opts.verbose)
	check that the list of runs is not empty More...

list	mytuple = []

	opts

	out

	output = subprocess.check_output([homedir+"/.local/bin/brilcalc", "lumi", "-b", "STABLE BEAMS","-u", "/pb", "--begin", str(minRun),"--end",str(maxRun),"--output-style","csv"])
	using normtag output = subprocess.check_output([homedir+"/.local/bin/brilcalc", "lumi", "-b", "STABLE BEAMS", "--normtag","/cvmfs/cms-bril.cern.ch/cms-lumi-pog/Normtags/normtag_PHYSICS.json", "-u", "/pb", "--begin", str(minRun),"--end",str(maxRun),"--output-style","csv"]) More...

	outputList = out.decode().split('\n')

	p

	parser = OptionParser(description=desc,version='%prog version 0.1')

	pool = multiprocessing.Pool(processes=20)

	q = Popen(cmd2 , shell=True, stdout=PIPE, stderr=PIPE)

def	runs = get_status_output("dasgoclient -query='run dataset="+opts.DATASET+"'",shell=True, stdout=PIPE, stderr=PIPE)[1].decode().split("\n")
	check first there is a valid grid proxy More...

	runToCache = line.split(",")[0].split(":")[0]

string	script

	scriptFile = open(scriptFileName,'w')

	scriptFileName = os.path.join(bashdir,"batchHarvester_"+key+"_"+str(count-1)+".sh")

	stderr

	stdout = dump)

string	submissionCommand = "condor_submit "+job_submit_file

string	theLumi = '1'

	USER = os.environ.get('USER')
	prepare the eos output directory More...

Detailed Description

Submits per run Primary Vertex Resoltion Alignment validation using the split vertex method,
usage:

submitPVResolutionJobs.py -i PVResolutionExample.ini -D /JetHT/Run2018C-TkAlMinBias-12Nov2019_UL2018-v2/ALCARECO

Function Documentation

◆ as_dict()

def submitPVResolutionJobs.as_dict ( config )

Definition at line 179 of file submitPVResolutionJobs.py.

179 def as_dict(config):

◆ batchScriptCERN()

def submitPVResolutionJobs.batchScriptCERN	(	theCMSSW_BASE,
		runindex,
		eosdir,
		lumiToRun,
		key,
		config
	)

Definition at line 190 of file submitPVResolutionJobs.py.

190 def batchScriptCERN(theCMSSW_BASE,runindex, eosdir,lumiToRun,key,config):

◆ check_proxy()

def submitPVResolutionJobs.check_proxy ( )

Definition at line 49 of file submitPVResolutionJobs.py.

49 def check_proxy():

Referenced by forward_proxy().

◆ forward_proxy()

def submitPVResolutionJobs.forward_proxy ( rundir )

Definition at line 62 of file submitPVResolutionJobs.py.

62 def forward_proxy(rundir):

References check_proxy(), and print().

◆ get_status_output()

def submitPVResolutionJobs.get_status_output	(	*	args,
		**	kwargs
	)

Definition at line 42 of file submitPVResolutionJobs.py.

42 def get_status_output(*args, **kwargs):

◆ getFilesForRun()

def submitPVResolutionJobs.getFilesForRun ( blob )

Definition at line 77 of file submitPVResolutionJobs.py.

77 def getFilesForRun(blob):

◆ getLuminosity()

def submitPVResolutionJobs.getLuminosity	(	homedir,
		minRun,
		maxRun,
		isRunBased,
		verbose
	)

Definition at line 127 of file submitPVResolutionJobs.py.

127 def getLuminosity(homedir,minRun,maxRun,isRunBased,verbose):

◆ isInJSON()

def submitPVResolutionJobs.isInJSON	(	run,
		jsonfile
	)

Definition at line 168 of file submitPVResolutionJobs.py.

168 def isInJSON(run,jsonfile):

◆ main()

def submitPVResolutionJobs.main ( )

Definition at line 248 of file submitPVResolutionJobs.py.

248 def main():

◆ mkdir_eos()

def submitPVResolutionJobs.mkdir_eos ( out_path )

method to create recursively directories on EOS

Definition at line 226 of file submitPVResolutionJobs.py.

 def mkdir_eos(out_path):
     print("creating",out_path)
     newpath='/'
     for dir in out_path.split('/'):
         newpath=os.path.join(newpath,dir)
         # do not issue mkdir from very top of the tree
         if newpath.find('test_out') > 0:
             command="eos mkdir "+newpath
             p = subprocess.Popen(command,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             (out, err) = p.communicate()
             #print(out,err)
             p.wait()
  
     # now check that the directory exists
     command2="/afs/cern.ch/project/eos/installation/cms/bin/eos.select ls "+out_path
     p = subprocess.Popen(command2,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (out, err) = p.communicate()
     p.wait()
     if p.returncode !=0:
         print(out)
  

References print().

◆ write_HTCondor_submit_file()

def submitPVResolutionJobs.write_HTCondor_submit_file	(	path,
		name,
		nruns,
		proxy_path = `None`
	)

Definition at line 91 of file submitPVResolutionJobs.py.

91 def write_HTCondor_submit_file(path, name, nruns, proxy_path=None):

Variable Documentation

◆ author

submitPVResolutionJobs.__author__

private

Definition at line 11 of file submitPVResolutionJobs.py.

◆ copyright

submitPVResolutionJobs.__copyright__

private

Definition at line 12 of file submitPVResolutionJobs.py.

◆ credits

submitPVResolutionJobs.__credits__

private

Definition at line 13 of file submitPVResolutionJobs.py.

◆ email

submitPVResolutionJobs.__email__

private

Definition at line 16 of file submitPVResolutionJobs.py.

◆ license

submitPVResolutionJobs.__license__

private

Definition at line 14 of file submitPVResolutionJobs.py.

◆ maintainer

submitPVResolutionJobs.__maintainer__

private

Definition at line 15 of file submitPVResolutionJobs.py.

◆ version

submitPVResolutionJobs.__version__

private

Definition at line 17 of file submitPVResolutionJobs.py.

◆ action

submitPVResolutionJobs.action

Definition at line 253 of file submitPVResolutionJobs.py.

◆ args

submitPVResolutionJobs.args

Definition at line 261 of file submitPVResolutionJobs.py.

◆ bashdir

submitPVResolutionJobs.bashdir = os.path.join(cwd,"BASH")

Definition at line 309 of file submitPVResolutionJobs.py.

◆ cmd2

string submitPVResolutionJobs.cmd2 = ' dasgoclient -limit=0 -query \'file run='+blob[0]+' dataset='+blob[1]+'\''

returns the list of list files associated with a given dataset for a certain run

Definition at line 83 of file submitPVResolutionJobs.py.

◆ config

submitPVResolutionJobs.config = ConfigParser.ConfigParser()

parse the configuration file

Definition at line 281 of file submitPVResolutionJobs.py.

Referenced by DTBtiChip.acceptMask(), DTBtiChip.computeEqs(), DTBtiChip.computeSums(), PhysicsTools::VarProcessor.configure(), TwoBodyDecayTrajectory.construct(), DualBzeroReferenceTrajectory.construct(), DualReferenceTrajectory.construct(), TwoBodyDecayTrajectoryFactory.constructTrajectories(), edm::ServiceRegistry.createContaining(), egPM::Param< reco::ElectronSeed >.createParamBin_(), Mixing2DB.endJob(), CTPPSOpticalFunctionsESSource.fillDescriptions(), CTPPSProtonProducer.fillDescriptions(), DTBtiChip.findTrig(), getHLTConfigData(), edm::Event.getProcessParameterSet(), edm::ScheduleItems.initSchedule(), DTBtiChip.keepTrig(), DTBtiChip.keepTrigPatt(), pos::PixelConfigFile.makeKey(), QualityTester.makeQCriterion(), edm::ProblemTracker.ProblemTracker(), pat::PATTriggerProducer.produce(), CSCTFConfigProducer.produceL1MuCSCTFConfigurationRcd(), FWConfigurationManager.readFromOldFile(), edm::MixingModule.reload(), edm::MixingWorker< T >.reload(), CmsShowMainBase.reloadConfiguration(), cond::Utilities.run(), PhysicsTools::MVAComputer.setup(), edm::WorkerManager.setupResolvers(), FWConfigurationManager.to(), BzeroReferenceTrajectoryFactory.trajectories(), DualTrajectoryFactory.trajectories(), DualBzeroTrajectoryFactory.trajectories(), and ReferenceTrajectoryFactory.trajectories().

◆ CopyRights

submitPVResolutionJobs.CopyRights

Definition at line 35 of file submitPVResolutionJobs.py.

◆ count

int submitPVResolutionJobs.count = pool.map(getFilesForRun,mytuple)

Definition at line 352 of file submitPVResolutionJobs.py.

◆ cwd

submitPVResolutionJobs.cwd = os.getcwd()

Definition at line 308 of file submitPVResolutionJobs.py.

◆ default

submitPVResolutionJobs.default

Definition at line 253 of file submitPVResolutionJobs.py.

◆ desc

string submitPVResolutionJobs.desc = """This is a description of %prog."""

Definition at line 251 of file submitPVResolutionJobs.py.

◆ dest

submitPVResolutionJobs.dest

Definition at line 253 of file submitPVResolutionJobs.py.

◆ dictionary

dictionary submitPVResolutionJobs.dictionary = {}

Definition at line 181 of file submitPVResolutionJobs.py.

◆ eosdir

submitPVResolutionJobs.eosdir = os.path.join("/store/group/alca_trackeralign",USER,"test_out",opts.taskname)

Definition at line 272 of file submitPVResolutionJobs.py.

◆ err

submitPVResolutionJobs.err

Definition at line 85 of file submitPVResolutionJobs.py.

◆ file_info

submitPVResolutionJobs.file_info = dict(zip(runs, count))

Definition at line 353 of file submitPVResolutionJobs.py.

◆ files

submitPVResolutionJobs.files = file_info[run]

Definition at line 373 of file submitPVResolutionJobs.py.

Referenced by check_files(), edm::service::ThreadSafeLogMessageLoggerScribe.configure_errorlog_new(), TrackerDpgAnalysis.delay(), HcalCalibrator.endJob(), JsonOutputProducer::JsonConfigurationBlock.extractFileBase(), fillABIOFiles(), fillABSRPFiles(), fillABTTFFiles(), MillePedeAlignmentAlgorithm.getExistingFormattedFiles(), SiStripDbParams.inputDcuInfoXmlFiles(), SiStripDbParams.inputFecXmlFiles(), SiStripDbParams.inputFedXmlFiles(), SiStripDbParams.inputModuleXmlFiles(), HcalPatternSource.loadPatterns(), main(), dqm::DQMFileSaverOnline.makeSnapshot(), PlotCombiner(), CTPPSRPAlignmentCorrectionsDataESSourceXML.PrepareSequence(), cms::DDDetector.process(), JsonOutputProducer::JsonConfigurationBlock.serialise(), SiStripCommissioningOfflineClient.setInputFiles(), and MillePedeAlignmentAlgorithm.terminate().

◆ help

submitPVResolutionJobs.help

Definition at line 253 of file submitPVResolutionJobs.py.

◆ HOME

submitPVResolutionJobs.HOME = os.environ.get('HOME')

Definition at line 271 of file submitPVResolutionJobs.py.

◆ input_CMSSW_BASE

submitPVResolutionJobs.input_CMSSW_BASE = os.environ.get('CMSSW_BASE')

Definition at line 266 of file submitPVResolutionJobs.py.

◆ inputDict

def submitPVResolutionJobs.inputDict = as_dict(config)

Definition at line 287 of file submitPVResolutionJobs.py.

◆ job_submit_file

def submitPVResolutionJobs.job_submit_file = os.path.join(path, "job_"+name+".submit")

\
+x509userproxy        = "{proxy:s}"

Definition at line 115 of file submitPVResolutionJobs.py.

◆ job_submit_template

string submitPVResolutionJobs.job_submit_template

Initial value:

 = """\
 universe              = vanilla
 executable            = {script:s}
 output                = {jobm:s}/{out:s}.out
 error                 = {jobm:s}/{out:s}.err
 log                   = {jobm:s}/{out:s}.log
 transfer_output_files = ""
 +JobFlavour           = "{flavour:s}"
 queue {njobs:s}
 """

Writes 'job.submit' file in `path`.
Arguments:
- `path`: job directory
- `script`: script to be executed
- `proxy_path`: path to proxy (only used in case of requested proxy forward)

Definition at line 100 of file submitPVResolutionJobs.py.

◆ jsonDATA

submitPVResolutionJobs.jsonDATA = json.load(myJSON)

Definition at line 172 of file submitPVResolutionJobs.py.

◆ key

submitPVResolutionJobs.key = key.split(":", 1)[1]

prepare the HTCondor submission files and eventually submit them

Definition at line 398 of file submitPVResolutionJobs.py.

◆ listOfFiles

string submitPVResolutionJobs.listOfFiles = '['

Definition at line 376 of file submitPVResolutionJobs.py.

◆ local_proxy

submitPVResolutionJobs.local_proxy = subprocess.check_output(["voms-proxy-info", "--path"]).strip()

Forward proxy to location visible from the batch system.
Arguments:
- `rundir`: directory for storing the forwarded proxy

Definition at line 73 of file submitPVResolutionJobs.py.

◆ lumimask

def submitPVResolutionJobs.lumimask = inputDict["Input"]["lumimask"]

Definition at line 331 of file submitPVResolutionJobs.py.

◆ lumiToCache

submitPVResolutionJobs.lumiToCache = line.split(",")[-1].replace("\r", "")

Definition at line 158 of file submitPVResolutionJobs.py.

◆ myCachedLumi

dictionary submitPVResolutionJobs.myCachedLumi = {}

Expects something like
+-------+------+--------+--------+-------------------+------------------+
| nfill | nrun | nls    | ncms   | totdelivered(/fb) | totrecorded(/fb) |
+-------+------+--------+--------+-------------------+------------------+
| 73    | 327  | 142418 | 138935 | 19.562            | 18.036           |
+-------+------+--------+--------+-------------------+------------------+
And extracts the total recorded luminosity (/b).

Definition at line 137 of file submitPVResolutionJobs.py.

◆ myLumiDB

def submitPVResolutionJobs.myLumiDB = getLuminosity(HOME,runs[0],runs[-1],True,opts.verbose)

check that the list of runs is not empty

Definition at line 326 of file submitPVResolutionJobs.py.

◆ mytuple

list submitPVResolutionJobs.mytuple = []

Definition at line 334 of file submitPVResolutionJobs.py.

◆ opts

submitPVResolutionJobs.opts

Definition at line 261 of file submitPVResolutionJobs.py.

◆ out

submitPVResolutionJobs.out

Definition at line 85 of file submitPVResolutionJobs.py.

◆ output

submitPVResolutionJobs.output = subprocess.check_output([homedir+"/.local/bin/brilcalc", "lumi", "-b", "STABLE BEAMS","-u", "/pb", "--begin", str(minRun),"--end",str(maxRun),"--output-style","csv"])

using normtag output = subprocess.check_output([homedir+"/.local/bin/brilcalc", "lumi", "-b", "STABLE BEAMS", "--normtag","/cvmfs/cms-bril.cern.ch/cms-lumi-pog/Normtags/normtag_PHYSICS.json", "-u", "/pb", "--begin", str(minRun),"--end",str(maxRun),"--output-style","csv"])

no normtag

Definition at line 146 of file submitPVResolutionJobs.py.

◆ outputList

submitPVResolutionJobs.outputList = out.decode().split('\n')

Definition at line 86 of file submitPVResolutionJobs.py.

Referenced by optutl::CommandLineParser._getSectionFiles().

◆ p

submitPVResolutionJobs.p

Definition at line 44 of file submitPVResolutionJobs.py.

◆ parser

submitPVResolutionJobs.parser = OptionParser(description=desc,version='%prog version 0.1')

Definition at line 252 of file submitPVResolutionJobs.py.

◆ pool

submitPVResolutionJobs.pool = multiprocessing.Pool(processes=20)

Definition at line 351 of file submitPVResolutionJobs.py.

Referenced by FWDialogBuilder.addLabel(), fireworks.boldGC(), CmsShowCommonPopup.CmsShowCommonPopup(), CentralityTableProducer.endJob(), tensorflow::NoThreadPool.instance(), tensorflow::TBBThreadPool.instance(), fireworks.italicGC(), and CmsShowMainFrame.showFWorksInfo().

◆ q

submitPVResolutionJobs.q = Popen(cmd2 , shell=True, stdout=PIPE, stderr=PIPE)

Definition at line 84 of file submitPVResolutionJobs.py.

◆ runs

def submitPVResolutionJobs.runs = get_status_output("dasgoclient -query='run dataset="+opts.DATASET+"'",shell=True, stdout=PIPE, stderr=PIPE)[1].decode().split("\n")

check first there is a valid grid proxy

Definition at line 297 of file submitPVResolutionJobs.py.

◆ runToCache

submitPVResolutionJobs.runToCache = line.split(",")[0].split(":")[0]

Definition at line 157 of file submitPVResolutionJobs.py.

◆ script

string submitPVResolutionJobs.script

Initial value:

 =  """#!/bin/bash
 source /afs/cern.ch/cms/caf/setup.sh
 CMSSW_DIR={CMSSW_BASE_DIR}/src/Alignment/OfflineValidation/test
 echo "the mother directory is $CMSSW_DIR"
 export X509_USER_PROXY=$CMSSW_DIR/.user_proxy
 #OUT_DIR=$CMSSW_DIR/harvest ## for local storage
 OUT_DIR={MYDIR}
 LOG_DIR=$CMSSW_DIR/out
 LXBATCH_DIR=`pwd`  
 cd $CMSSW_DIR
 eval `scram runtime -sh`
 cd $LXBATCH_DIR 
 cp -pr $CMSSW_DIR/cfg/PrimaryVertexResolution_{KEY}_{runindex}_cfg.py .
 cmsRun PrimaryVertexResolution_{KEY}_{runindex}_cfg.py GlobalTag={GT} lumi={LUMITORUN} {REC} {EXT} >& log_{KEY}_run{runindex}.out
 ls -lh . 
 #for payloadOutput in $(ls *root ); do cp $payloadOutput $OUT_DIR/pvresolution_{KEY}_{runindex}.root ; done 
 for payloadOutput in $(ls *root ); do xrdcp -f $payloadOutput root://eoscms/$OUT_DIR/pvresolution_{KEY}_{runindex}.root ; done
 tar czf log_{KEY}_run{runindex}.tgz log_{KEY}_run{runindex}.out  
 for logOutput in $(ls *tgz ); do cp $logOutput $LOG_DIR/ ; done 
 """.format(CMSSW_BASE_DIR=theCMSSW_BASE,
            runindex=runindex,
            MYDIR=eosdir,
            KEY=key,
            LUMITORUN=lumiToRun,
            GT=config['globaltag'],
            EXT="external="+config['external'] if 'external' in config.keys() else "",
            REC="records="+config['records'] if 'records' in config.keys() else "")

prepare the batch script, to run on HTCondor

Definition at line 193 of file submitPVResolutionJobs.py.

Referenced by gen::BaseHadronizer.generateLHE(), data_formats.objects_to_dicts(), data_formats.query(), run_script(), data_formats.to_array_of_dicts(), and data_formats.to_datatables().

◆ scriptFile

submitPVResolutionJobs.scriptFile = open(scriptFileName,'w')

Definition at line 407 of file submitPVResolutionJobs.py.

◆ scriptFileName

submitPVResolutionJobs.scriptFileName = os.path.join(bashdir,"batchHarvester_"+key+"_"+str(count-1)+".sh")

Definition at line 406 of file submitPVResolutionJobs.py.

◆ stderr

submitPVResolutionJobs.stderr

Definition at line 45 of file submitPVResolutionJobs.py.

Referenced by spu.create_dir(), getInfoFromDAQ(), MyAlloc(), MyFree(), spu.Untar(), fwlite::Scanner< Collection >.wantMore(), and spu.zerr().

◆ stdout

submitPVResolutionJobs.stdout = dump)

Check if GRID proxy has been initialized.

Definition at line 45 of file submitPVResolutionJobs.py.

◆ submissionCommand

string submitPVResolutionJobs.submissionCommand = "condor_submit "+job_submit_file

Definition at line 423 of file submitPVResolutionJobs.py.

◆ theLumi

string submitPVResolutionJobs.theLumi = '1'

Definition at line 383 of file submitPVResolutionJobs.py.

◆ USER

submitPVResolutionJobs.USER = os.environ.get('USER')

prepare the eos output directory

Definition at line 270 of file submitPVResolutionJobs.py.

Functions

Variables

Detailed Description

Function Documentation

◆ as_dict()

◆ batchScriptCERN()

◆ check_proxy()

◆ forward_proxy()

◆ get_status_output()

◆ getFilesForRun()

◆ getLuminosity()

◆ isInJSON()

◆ main()

◆ mkdir_eos()

◆ write_HTCondor_submit_file()

Variable Documentation

◆ __author__

◆ __copyright__

◆ __credits__

◆ __email__

◆ __license__

◆ __maintainer__

◆ __version__

◆ action

◆ args

◆ bashdir

◆ cmd2

◆ config

◆ CopyRights

◆ count

◆ cwd

◆ default

◆ desc

◆ dest

◆ dictionary

◆ eosdir

◆ err

◆ file_info

◆ files

◆ help

◆ HOME

◆ input_CMSSW_BASE

◆ inputDict

◆ job_submit_file

◆ job_submit_template

◆ jsonDATA

◆ key

◆ listOfFiles

◆ local_proxy

◆ lumimask

◆ lumiToCache

◆ myCachedLumi

◆ myLumiDB

◆ mytuple

◆ opts

◆ out

◆ output

◆ outputList

◆ p

◆ parser

◆ pool

◆ q

◆ runs

◆ runToCache

◆ script

◆ scriptFile

◆ scriptFileName

◆ stderr

◆ stdout

◆ submissionCommand

◆ theLumi

◆ USER

◆ author

◆ copyright

◆ credits

◆ email

◆ license

◆ maintainer

◆ version