harvestRelVal Namespace Reference

Functions
def	append_sample_mcrab (dsetp, fout)
def	check_dbs ()
def	check_dset ()
def	check_nevts_dset (dset)
def	create_crab (ds)
def	create_harvest (ds)
def	create_mcrab (set, fcrab, fout)
def	find_dqmref (ds)
def	get_cond_from_dsetpath (ds)
def	get_name_from_dsetpath (ds)
def	make_dbs_list (dbslf)
def	make_dqmname (s)
def	make_harv_name (dset)
def	print_def ()
def	read_ds_file ()

Function Documentation

def harvestRelVal.append_sample_mcrab	(		dsetp,
			fout
	)

Definition at line 216 of file harvestRelVal.py.

References find_dqmref(), get_name_from_dsetpath(), make_dqmname(), make_harv_name(), and edm.print().

Referenced by create_mcrab().

def append_sample_mcrab(dsetp, fout):
    dqm = make_dqmname(dsetp)
    sample = get_name_from_dsetpath(dsetp)
    hf = make_harv_name(dsetp)
    if not os.path.exists(hf) :
        print('problem creating multicrab, file', hf, 'does not exist')
        sys.exit(17)
    fout.write('\n\n[' + sample + ']')
    fout.write('\nCMSSW.pset=' + hf)
    fout.write('\nCMSSW.datasetpath=' + dsetp)
    fout.write('\nCMSSW.output_file=' + dqm)

    dqmref = find_dqmref(dsetp);
    if not dqmref == 'NONE' : 
        fout.write('\nUSER.additional_input_files=' + dqmref)

def harvestRelVal.check_dbs

(

)

Definition at line 25 of file harvestRelVal.py.

Referenced by create_crab().

def check_dbs():
    if os.getenv('DBSCMD_HOME','NOTSET') == 'NOTSET' :
        return 0
    return 1

def harvestRelVal.check_dset

(

)

Definition at line 97 of file harvestRelVal.py.

References check_nevts_dset(), get_cond_from_dsetpath(), list(), and edm.print().

Referenced by create_crab().

def check_dset() :
   #check cmssw consistency
   for s in dsetpaths:
       if s.find(cmssw_ver) == -1 :
           dsetpaths.remove(s)        
           print('Inconsistency found with datset and cmssw version (', cmssw_ver, ')' \
                 ': \t ', s, ' has been removed.')
   #check conditions from dsetname
   for s in dsetpaths[:]: #nb:need to make a copy here!
       cond = get_cond_from_dsetpath(s)
       if cond  == 0 : 
           dsetpaths.remove(s)        
   #check list size
   nSamples = len(dsetpaths)
   if nSamples == 0 :
       print("Empty input list, exit.")
       sys.exit(12)
   else :
       print('Processing', nSamples, 'data sets.')
   #check event numbers
   nSampleEvts = list()
   for s in dsetpaths:
       nSampleEvts.append(check_nevts_dset(s))
   print('number of events per dataset:', nSampleEvts)

def harvestRelVal.check_nevts_dset

(

dset

)

Definition at line 30 of file harvestRelVal.py.

References str.

Referenced by check_dset().

def check_nevts_dset(dset):
    if not is_dbs :
        return -1
    ntot=0
    for afile in api.listFiles(path=str(dset)):
        nevts = afile['NumberOfEvents']
        ntot += nevts
        #print "  %s" % afile['LogicalFileName']
    return ntot  

def harvestRelVal.create_crab

(

ds

)

Definition at line 232 of file harvestRelVal.py.

References check_dbs(), check_dset(), create_harvest(), create_mcrab(), list(), make_dbs_list(), make_dqmname(), make_harv_name(), edm.print(), print_def(), and read_ds_file().

def create_crab(ds) :
    dqmout = make_dqmname(ds)
    hf = make_harv_name(ds)
    out = open(f_crab, 'w')
    out.write(crab_block)
    out.write('\npset=' + hf)
    out.write('datasetpath=' + ds)
    out.write('\noutput_file=' + dqmout)
    out.close()

def harvestRelVal.create_harvest

(

ds

)

Definition at line 156 of file harvestRelVal.py.

References find_dqmref(), get_cond_from_dsetpath(), make_harv_name(), and edm.print().

Referenced by create_crab().

def create_harvest(ds) :
    raw_cmsdriver = "cmsDriver.py harvest -s HARVESTING:validationHarvesting --mc  --conditions FrontierConditions_GlobalTag,STARTUP_30X::All --harvesting AtJobEnd --no_exec -n -1"
    cmsdriver = raw_cmsdriver
    cond = get_cond_from_dsetpath(ds)
    if cond == 0 :
        print('unexpected problem with conditions')
        sys.exit(50)
    cmsdriver = cmsdriver.replace('30X',cond)
    fin_name="harvest_HARVESTING_STARTUP.py"
    if ds.find('IDEAL') != -1 :
        cmsdriver = cmsdriver.replace('STARTUP','IDEAL')
        fin_name = fin_name.replace('STARTUP','IDEAL')
    if ds.find('FastSim') != -1:
        cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingFS')
    if ds.find('PileUp') != -1:
        cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingPU')

    #print "=>", cmsdriver, " fs?", ds.find('FastSim')
    if os.path.exists(fin_name) : 
        os.system("rm " + fin_name)
    print("executing cmsdriver command:\n\t", cmsdriver)
    os.system(cmsdriver)
    if not os.path.exists(fin_name) : 
        print('problem with cmsdriver file name')
        sys.exit(40)
    os.system("touch " + fin_name)
    hf = make_harv_name(ds)
    os.system('mv ' + fin_name + " " + hf)
    out = open(hf, 'a')
    out.write("\n\n##additions to cmsDriver output \n")
    out.write("process.dqmSaver.workflow = '" + ds + "'\n")
    if is_dbs :
        out.write("process.source.fileNames = cms.untracked.vstring(\n")
        for afile in api.listFiles(path=ds):
            out.write("  '%s',\n" % afile['LogicalFileName'])
        out.write(")\n")

    dqmref = find_dqmref(ds);
    if not dqmref == 'NONE' : 
        out.write("process.DQMStore.referenceFileName = '" + dqmref + "'\n")
        out.write("process.dqmSaver.referenceHandling = 'all'\n")

    out.close()

def harvestRelVal.create_mcrab	(		set,
			fcrab,
			fout
	)

Definition at line 200 of file harvestRelVal.py.

References append_sample_mcrab().

Referenced by create_crab().

def create_mcrab(set, fcrab, fout):
    out = open(fout, 'w')
    out.write('[MULTICRAB]')
    out.write('\ncfg=' + fcrab)
    out.write('\n\n[COMMON]')
    nevt = -1
    njob = 1
    out.write('\nCMSSW.total_number_of_events=' + (str)(nevt) )
    out.write('\nCMSSW.number_of_jobs=' + (str)(njob) )
    for s in set:
        append_sample_mcrab(s, out)
    out.close()    

def harvestRelVal.find_dqmref

(

ds

)

Definition at line 122 of file harvestRelVal.py.

References make_dqmname(), edm.print(), python.rootplot.root2matplotlib.replace(), and str.

Referenced by append_sample_mcrab(), and create_harvest().

def find_dqmref(ds) :
    if not do_reference :
        return 'NONE'
    cp = cmssw_ver[-1:]
    ip = (int)(cp) - 1
    ref_ver = cmssw_ver.replace(cp,str(ip))
    #print "cms:", cmssw_ver, " cp:", cp, " ip:", ip, " new_ver:", ref_ver  
    ref_dir = "/castor/cern.ch/user/n/nuno/relval/harvest/" + ref_ver + "/"
    ref_dsf = make_dqmname(ds.replace(cmssw_ver, ref_ver))
    gls = " | grep root | grep "
    #to accept crab appended _1.root in file names, nd skip versions/conditions
    gls += ref_dsf[:-25] 
    gls += "| awk '{print $9}' "
    #print "refds:", ref_dsf, " command: rfdir", ref_dir+gls
    command = "rfcp " + ref_dir  + "`rfdir " + ref_dir + gls + "` ."
    #print "command:", command
    os.system(command)
    tmpfile = "ref.txt"
    command = "ls -rtl *" + gls + " > " + tmpfile
    #print "command:", command
    os.system(command)
    the_ref = 'NONE'
    if os.path.exists(tmpfile) :
        fin = open(tmpfile,'r')
        ref = fin.readline().replace('\n','')
        #print "read ref:", ref, "exists?", os.path.exists(ref)
        fin.close()
        if os.path.exists(ref) :
            the_ref = ref
    else :
        the_ref = 'NONE'
    print("Found reference file:", the_ref)
    return the_ref

def harvestRelVal.get_cond_from_dsetpath

(

ds

)

Definition at line 48 of file harvestRelVal.py.

References spr.find(), edm.print(), and python.rootplot.root2matplotlib.replace().

Referenced by check_dset(), and create_harvest().

def get_cond_from_dsetpath(ds) :
    ca = ds.split('/')[2].replace(cmssw_ver+'_','').replace('IDEAL_','').replace('STARTUP_','').replace('_FastSim','')
    cb = ca[:ca.find('v')-1]
    if cb[0].find('3') == -1 or len(cb) > 3:
        print("problem extracting condition for", ds, " : ", cb, '(len:',len(cb),')')  
        if cb.find('31X') != -1:
            cb = '31X'
        elif cb.find('30X') != -1:
            cb = '30X'
        else:
            print("skipping", cb)
            return 0
        print("condition found:", cb)
    else :
        print("good condition for", ds, " : ", cb, '(len:',len(cb),')')      
    return cb

def harvestRelVal.get_name_from_dsetpath

(

ds

)

Definition at line 43 of file harvestRelVal.py.

References python.rootplot.root2matplotlib.replace().

Referenced by append_sample_mcrab(), and make_harv_name().

def get_name_from_dsetpath(ds):
    fs = ds.split('/')
    fa = fs[1].replace('RelVal','')
    return fa

def harvestRelVal.make_dbs_list

(

dbslf

)

Definition at line 66 of file harvestRelVal.py.

References edm.print().

Referenced by create_crab().

def make_dbs_list(dbslf) :
    if not is_dbs :
        return
    flis = open(dbslf,'w')
    for ads in api.listDatasetPaths() :
        if ads.find('RelVal') != -1 \
               or ads.find(cmssw_ver) != -1 \
               or ads.find("/GEN-SIM") != -1 : 
#               and ads.find("/GEN-SIM-RECO") != -1 : 
            flis.write(ads + '\n')
    flis.close()
    print('Generated dataset list', dbslf, 'from dbs.') 
    #exampe:
    #dbs lsd --path=/RelVal*/CMSSW_3_1_0_pre5*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet > mylist.txt
    #dbslsd = "dbs lsd --path=/RelVal*/" + cmssw_ver + "*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
    #os.system( '`' + dbslsd + ' > ' + dbslf + '`')

def harvestRelVal.make_dqmname

(

s

)

Definition at line 40 of file harvestRelVal.py.

Referenced by append_sample_mcrab(), create_crab(), and find_dqmref().

def make_dqmname(s):
    return  'DQM_V0001_R000000001' + s.replace('/','__') + '.root' 

def harvestRelVal.make_harv_name

(

dset

)

Definition at line 213 of file harvestRelVal.py.

References get_name_from_dsetpath().

Referenced by append_sample_mcrab(), create_crab(), and create_harvest().

def make_harv_name(dset) :
    return 'harvest_' + get_name_from_dsetpath(dset) + '.py' 

def harvestRelVal.print_def

(

)

Definition at line 18 of file harvestRelVal.py.

References edm.print().

Referenced by create_crab().

def print_def():
    print("Usage:", sys.argv[0], "[list_of_processes]")
    print("Examples:")
    print("harvestRelVal.py")
    print("harvestRelVal.py /RelValTTbar/CMSSW_3_1_0_pre4_STARTUP_30X_v1/GEN-SIM-RECO")
    print("harvestRelVal.py <dataset_list.txt>")

def harvestRelVal.read_ds_file

(

)

Definition at line 83 of file harvestRelVal.py.

References edm.print().

Referenced by create_crab().

def read_ds_file() :
    if not os.path.exists(dsfile) :
        print("problem reading file", dsfile)
        sys.exit(30)
    fin = open(dsfile,'r')
    for dset in fin.readlines(): 
        d = dset.replace('\n','')
        if d.find('#') == -1 :
            dsetpaths.append(d)
        else :
            print('skipping:', d)
    fin.close()
    print('Using data set list in ', dsfile)