harvestRelVal Namespace Reference

Functions
def	append_sample_mcrab (dsetp, fout)
def	check_dbs ()
def	check_dset ()
def	check_nevts_dset (dset)
def	create_crab (ds)
def	create_harvest (ds)
def	create_mcrab (set, fcrab, fout)
def	find_dqmref (ds)
def	get_cond_from_dsetpath (ds)
def	get_name_from_dsetpath (ds)
def	make_dbs_list (dbslf)
def	make_dqmname (s)
def	make_harv_name (dset)
def	print_def ()
def	read_ds_file ()

Variables
	api
	argin
	args
	cmssw_ver
	crab_block
	do_reference
	dsetpaths
	dsfile
	dslproc
	f_crab
	Create harvest.py template. More...
	f_multi_crab
	Create harvest_n.py for individual datasets. More...
	harvfilelist
	Print what has been created. More...
	input_type
	is_dbs
	optManager
	opts

Function Documentation

append_sample_mcrab()

def harvestRelVal.append_sample_mcrab	(		dsetp,
			fout
	)

Definition at line 215 of file harvestRelVal.py.

References find_dqmref(), get_name_from_dsetpath(), make_dqmname(), make_harv_name(), and print().

Referenced by create_mcrab().

def append_sample_mcrab(dsetp, fout):
    dqm = make_dqmname(dsetp)
    sample = get_name_from_dsetpath(dsetp)
    hf = make_harv_name(dsetp)
    if not os.path.exists(hf) :
        print('problem creating multicrab, file', hf, 'does not exist')
        sys.exit(17)
    fout.write('\n\n[' + sample + ']')
    fout.write('\nCMSSW.pset=' + hf)
    fout.write('\nCMSSW.datasetpath=' + dsetp)
    fout.write('\nCMSSW.output_file=' + dqm)

    dqmref = find_dqmref(dsetp);
    if not dqmref == 'NONE' : 
        fout.write('\nUSER.additional_input_files=' + dqmref)

check_dbs()

def harvestRelVal.check_dbs

(

)

Definition at line 25 of file harvestRelVal.py.

def check_dbs():
    if os.getenv('DBSCMD_HOME','NOTSET') == 'NOTSET' :
        return 0
    return 1

check_dset()

def harvestRelVal.check_dset

(

)

Definition at line 97 of file harvestRelVal.py.

References check_nevts_dset(), get_cond_from_dsetpath(), and print().

def check_dset() :
   #check cmssw consistency
   for s in dsetpaths:
       if s.find(cmssw_ver) == -1 :
           dsetpaths.remove(s)        
           print('Inconsistency found with datset and cmssw version (', cmssw_ver, ')' \
                 ': \t ', s, ' has been removed.')
   #check conditions from dsetname
   for s in dsetpaths[:]: #nb:need to make a copy here!
       cond = get_cond_from_dsetpath(s)
       if cond  == 0 : 
           dsetpaths.remove(s)        
   #check list size
   nSamples = len(dsetpaths)
   if nSamples == 0 :
       print("Empty input list, exit.")
       sys.exit(12)
   else :
       print('Processing', nSamples, 'data sets.')
   #check event numbers
   nSampleEvts = list()
   for s in dsetpaths:
       nSampleEvts.append(check_nevts_dset(s))
   print('number of events per dataset:', nSampleEvts)

check_nevts_dset()

def harvestRelVal.check_nevts_dset

(

dset

)

Definition at line 30 of file harvestRelVal.py.

References str.

Referenced by check_dset().

def check_nevts_dset(dset):
    if not is_dbs :
        return -1
    ntot=0
    for afile in api.listFiles(path=str(dset)):
        nevts = afile['NumberOfEvents']
        ntot += nevts
        #print "  %s" % afile['LogicalFileName']
    return ntot  

create_crab()

def harvestRelVal.create_crab

(

ds

)

Definition at line 231 of file harvestRelVal.py.

References make_dqmname(), and make_harv_name().

def create_crab(ds) :
    dqmout = make_dqmname(ds)
    hf = make_harv_name(ds)
    out = open(f_crab, 'w')
    out.write(crab_block)
    out.write('\npset=' + hf)
    out.write('datasetpath=' + ds)
    out.write('\noutput_file=' + dqmout)
    out.close()

create_harvest()

def harvestRelVal.create_harvest

(

ds

)

Definition at line 156 of file harvestRelVal.py.

References find_dqmref(), get_cond_from_dsetpath(), make_harv_name(), and print().

def create_harvest(ds) :
    raw_cmsdriver = "cmsDriver.py harvest -s HARVESTING:validationHarvesting --mc  --conditions FrontierConditions_GlobalTag,STARTUP_30X::All --harvesting AtJobEnd --no_exec -n -1"
    cmsdriver = raw_cmsdriver
    cond = get_cond_from_dsetpath(ds)
    if cond == 0 :
        print('unexpected problem with conditions')
        sys.exit(50)
    cmsdriver = cmsdriver.replace('30X',cond)
    fin_name="harvest_HARVESTING_STARTUP.py"
    if ds.find('IDEAL') != -1 :
        cmsdriver = cmsdriver.replace('STARTUP','IDEAL')
        fin_name = fin_name.replace('STARTUP','IDEAL')
    if ds.find('FastSim') != -1:
        cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingFS')
    if ds.find('PileUp') != -1:
        cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingPU')

    #print "=>", cmsdriver, " fs?", ds.find('FastSim')
    if os.path.exists(fin_name) : 
        os.system("rm " + fin_name)
    print("executing cmsdriver command:\n\t", cmsdriver)
    os.system(cmsdriver)
    if not os.path.exists(fin_name) : 
        print('problem with cmsdriver file name')
        sys.exit(40)
    os.system("touch " + fin_name)
    hf = make_harv_name(ds)
    os.system('mv ' + fin_name + " " + hf)
    out = open(hf, 'a')
    out.write("\n\n##additions to cmsDriver output \n")
    out.write("process.dqmSaver.workflow = '" + ds + "'\n")
    if is_dbs :
        out.write("process.source.fileNames = cms.untracked.vstring(\n")
        for afile in api.listFiles(path=ds):
            out.write("  '%s',\n" % afile['LogicalFileName'])
        out.write(")\n")

    dqmref = find_dqmref(ds);
    if not dqmref == 'NONE' : 
        out.write("process.dqmSaver.referenceHandling = 'all'\n")

    out.close()

create_mcrab()

def harvestRelVal.create_mcrab	(		set,
			fcrab,
			fout
	)

Definition at line 199 of file harvestRelVal.py.

References append_sample_mcrab().

def create_mcrab(set, fcrab, fout):
    out = open(fout, 'w')
    out.write('[MULTICRAB]')
    out.write('\ncfg=' + fcrab)
    out.write('\n\n[COMMON]')
    nevt = -1
    njob = 1
    out.write('\nCMSSW.total_number_of_events=' + (str)(nevt) )
    out.write('\nCMSSW.number_of_jobs=' + (str)(njob) )
    for s in set:
        append_sample_mcrab(s, out)
    out.close()    

find_dqmref()

def harvestRelVal.find_dqmref

(

ds

)

Definition at line 122 of file harvestRelVal.py.

References make_dqmname(), print(), python.rootplot.root2matplotlib.replace(), and str.

Referenced by append_sample_mcrab(), and create_harvest().

def find_dqmref(ds) :
    if not do_reference :
        return 'NONE'
    cp = cmssw_ver[-1:]
    ip = (int)(cp) - 1
    ref_ver = cmssw_ver.replace(cp,str(ip))
    #print "cms:", cmssw_ver, " cp:", cp, " ip:", ip, " new_ver:", ref_ver  
    ref_dir = "/castor/cern.ch/user/n/nuno/relval/harvest/" + ref_ver + "/"
    ref_dsf = make_dqmname(ds.replace(cmssw_ver, ref_ver))
    gls = " | grep root | grep "
    #to accept crab appended _1.root in file names, nd skip versions/conditions
    gls += ref_dsf[:-25] 
    gls += "| awk '{print $9}' "
    #print "refds:", ref_dsf, " command: rfdir", ref_dir+gls
    command = "rfcp " + ref_dir  + "`rfdir " + ref_dir + gls + "` ."
    #print "command:", command
    os.system(command)
    tmpfile = "ref.txt"
    command = "ls -rtl *" + gls + " > " + tmpfile
    #print "command:", command
    os.system(command)
    the_ref = 'NONE'
    if os.path.exists(tmpfile) :
        fin = open(tmpfile,'r')
        ref = fin.readline().replace('\n','')
        #print "read ref:", ref, "exists?", os.path.exists(ref)
        fin.close()
        if os.path.exists(ref) :
            the_ref = ref
    else :
        the_ref = 'NONE'
    print("Found reference file:", the_ref)
    return the_ref

get_cond_from_dsetpath()

def harvestRelVal.get_cond_from_dsetpath

(

ds

)

Definition at line 48 of file harvestRelVal.py.

References spr.find(), print(), and python.rootplot.root2matplotlib.replace().

Referenced by check_dset(), and create_harvest().

def get_cond_from_dsetpath(ds) :
    ca = ds.split('/')[2].replace(cmssw_ver+'_','').replace('IDEAL_','').replace('STARTUP_','').replace('_FastSim','')
    cb = ca[:ca.find('v')-1]
    if cb[0].find('3') == -1 or len(cb) > 3:
        print("problem extracting condition for", ds, " : ", cb, '(len:',len(cb),')')  
        if cb.find('31X') != -1:
            cb = '31X'
        elif cb.find('30X') != -1:
            cb = '30X'
        else:
            print("skipping", cb)
            return 0
        print("condition found:", cb)
    else :
        print("good condition for", ds, " : ", cb, '(len:',len(cb),')')      
    return cb

get_name_from_dsetpath()

def harvestRelVal.get_name_from_dsetpath

(

ds

)

Definition at line 43 of file harvestRelVal.py.

References python.rootplot.root2matplotlib.replace().

Referenced by append_sample_mcrab(), and make_harv_name().

def get_name_from_dsetpath(ds):
    fs = ds.split('/')
    fa = fs[1].replace('RelVal','')
    return fa

make_dbs_list()

def harvestRelVal.make_dbs_list

(

dbslf

)

Definition at line 66 of file harvestRelVal.py.

References print().

def make_dbs_list(dbslf) :
    if not is_dbs :
        return
    flis = open(dbslf,'w')
    for ads in api.listDatasetPaths() :
        if ads.find('RelVal') != -1 \
               or ads.find(cmssw_ver) != -1 \
               or ads.find("/GEN-SIM") != -1 : 
#               and ads.find("/GEN-SIM-RECO") != -1 : 
            flis.write(ads + '\n')
    flis.close()
    print('Generated dataset list', dbslf, 'from dbs.') 
    #exampe:
    #dbs lsd --path=/RelVal*/CMSSW_3_1_0_pre5*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet > mylist.txt
    #dbslsd = "dbs lsd --path=/RelVal*/" + cmssw_ver + "*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
    #os.system( '`' + dbslsd + ' > ' + dbslf + '`')

make_dqmname()

def harvestRelVal.make_dqmname

(

s

)

Definition at line 40 of file harvestRelVal.py.

Referenced by append_sample_mcrab(), create_crab(), and find_dqmref().

def make_dqmname(s):
    return  'DQM_V0001_R000000001' + s.replace('/','__') + '.root' 

make_harv_name()

def harvestRelVal.make_harv_name

(

dset

)

Definition at line 212 of file harvestRelVal.py.

References get_name_from_dsetpath().

Referenced by append_sample_mcrab(), create_crab(), and create_harvest().

def make_harv_name(dset) :
    return 'harvest_' + get_name_from_dsetpath(dset) + '.py' 

print_def()

def harvestRelVal.print_def

(

)

Definition at line 18 of file harvestRelVal.py.

References print().

def print_def():
    print("Usage:", sys.argv[0], "[list_of_processes]")
    print("Examples:")
    print("harvestRelVal.py")
    print("harvestRelVal.py /RelValTTbar/CMSSW_3_1_0_pre4_STARTUP_30X_v1/GEN-SIM-RECO")
    print("harvestRelVal.py <dataset_list.txt>")

read_ds_file()

def harvestRelVal.read_ds_file

(

)

Definition at line 83 of file harvestRelVal.py.

References print().

def read_ds_file() :
    if not os.path.exists(dsfile) :
        print("problem reading file", dsfile)
        sys.exit(30)
    fin = open(dsfile,'r')
    for dset in fin.readlines(): 
        d = dset.replace('\n','')
        if d.find('#') == -1 :
            dsetpaths.append(d)
        else :
            print('skipping:', d)
    fin.close()
    print('Using data set list in ', dsfile)

Variable Documentation

api

harvestRelVal.api

Definition at line 307 of file harvestRelVal.py.

argin

harvestRelVal.argin

Definition at line 269 of file harvestRelVal.py.

args

harvestRelVal.args

Definition at line 303 of file harvestRelVal.py.

cmssw_ver

harvestRelVal.cmssw_ver

Definition at line 310 of file harvestRelVal.py.

crab_block

harvestRelVal.crab_block

Definition at line 241 of file harvestRelVal.py.

do_reference

harvestRelVal.do_reference

Definition at line 271 of file harvestRelVal.py.

dsetpaths

harvestRelVal.dsetpaths

Definition at line 326 of file harvestRelVal.py.

dsfile

harvestRelVal.dsfile

Definition at line 270 of file harvestRelVal.py.

dslproc

harvestRelVal.dslproc

Definition at line 348 of file harvestRelVal.py.

f_crab

harvestRelVal.f_crab

Create harvest.py template.

Create crab.cfg template

Definition at line 358 of file harvestRelVal.py.

f_multi_crab

harvestRelVal.f_multi_crab

Create harvest_n.py for individual datasets.

Create multicrab.cfg

Definition at line 366 of file harvestRelVal.py.

harvfilelist

harvestRelVal.harvfilelist

Print what has been created.

Definition at line 371 of file harvestRelVal.py.

input_type

harvestRelVal.input_type

Definition at line 268 of file harvestRelVal.py.

is_dbs

harvestRelVal.is_dbs

Definition at line 293 of file harvestRelVal.py.

optManager

harvestRelVal.optManager

Definition at line 302 of file harvestRelVal.py.

opts

harvestRelVal.opts

Definition at line 303 of file harvestRelVal.py.