CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
Functions | Variables
harvestRelVal Namespace Reference

Functions

def append_sample_mcrab (dsetp, fout)
 
def check_dbs ()
 
def check_dset ()
 
def check_nevts_dset (dset)
 
def create_crab (ds)
 
def create_harvest (ds)
 
def create_mcrab (set, fcrab, fout)
 
def find_dqmref (ds)
 
def get_cond_from_dsetpath (ds)
 
def get_name_from_dsetpath (ds)
 
def make_dbs_list (dbslf)
 
def make_dqmname (s)
 
def make_harv_name (dset)
 
def print_def ()
 
def read_ds_file ()
 

Variables

 api
 
 argin
 
 args
 
 cmssw_ver
 
 crab_block
 
 do_reference
 
 dsetpaths
 
 dsfile
 
 dslproc
 
 f_crab
 Create harvest.py template. More...
 
 f_multi_crab
 Create harvest_n.py for individual datasets. More...
 
 harvfilelist
 Print what has been created. More...
 
 input_type
 
 is_dbs
 
 optManager
 
 opts
 

Function Documentation

◆ append_sample_mcrab()

def harvestRelVal.append_sample_mcrab (   dsetp,
  fout 
)

Definition at line 215 of file harvestRelVal.py.

References find_dqmref(), get_name_from_dsetpath(), make_dqmname(), make_harv_name(), and print().

Referenced by create_mcrab().

215 def append_sample_mcrab(dsetp, fout):
216  dqm = make_dqmname(dsetp)
217  sample = get_name_from_dsetpath(dsetp)
218  hf = make_harv_name(dsetp)
219  if not os.path.exists(hf) :
220  print('problem creating multicrab, file', hf, 'does not exist')
221  sys.exit(17)
222  fout.write('\n\n[' + sample + ']')
223  fout.write('\nCMSSW.pset=' + hf)
224  fout.write('\nCMSSW.datasetpath=' + dsetp)
225  fout.write('\nCMSSW.output_file=' + dqm)
226 
227  dqmref = find_dqmref(dsetp);
228  if not dqmref == 'NONE' :
229  fout.write('\nUSER.additional_input_files=' + dqmref)
230 
def get_name_from_dsetpath(ds)
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def make_harv_name(dset)
def append_sample_mcrab(dsetp, fout)
def find_dqmref(ds)
def make_dqmname(s)

◆ check_dbs()

def harvestRelVal.check_dbs ( )

Definition at line 25 of file harvestRelVal.py.

25 def check_dbs():
26  if os.getenv('DBSCMD_HOME','NOTSET') == 'NOTSET' :
27  return 0
28  return 1
29 

◆ check_dset()

def harvestRelVal.check_dset ( )

Definition at line 97 of file harvestRelVal.py.

References check_nevts_dset(), get_cond_from_dsetpath(), and print().

97 def check_dset() :
98  #check cmssw consistency
99  for s in dsetpaths:
100  if s.find(cmssw_ver) == -1 :
101  dsetpaths.remove(s)
102  print('Inconsistency found with datset and cmssw version (', cmssw_ver, ')' \
103  ': \t ', s, ' has been removed.')
104  #check conditions from dsetname
105  for s in dsetpaths[:]: #nb:need to make a copy here!
106  cond = get_cond_from_dsetpath(s)
107  if cond == 0 :
108  dsetpaths.remove(s)
109  #check list size
110  nSamples = len(dsetpaths)
111  if nSamples == 0 :
112  print("Empty input list, exit.")
113  sys.exit(12)
114  else :
115  print('Processing', nSamples, 'data sets.')
116  #check event numbers
117  nSampleEvts = list()
118  for s in dsetpaths:
119  nSampleEvts.append(check_nevts_dset(s))
120  print('number of events per dataset:', nSampleEvts)
121 
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def get_cond_from_dsetpath(ds)
def check_nevts_dset(dset)

◆ check_nevts_dset()

def harvestRelVal.check_nevts_dset (   dset)

Definition at line 30 of file harvestRelVal.py.

References str.

Referenced by check_dset().

30 def check_nevts_dset(dset):
31  if not is_dbs :
32  return -1
33  ntot=0
34  for afile in api.listFiles(path=str(dset)):
35  nevts = afile['NumberOfEvents']
36  ntot += nevts
37  #print " %s" % afile['LogicalFileName']
38  return ntot
39 
def check_nevts_dset(dset)
#define str(s)

◆ create_crab()

def harvestRelVal.create_crab (   ds)

Definition at line 231 of file harvestRelVal.py.

References make_dqmname(), and make_harv_name().

231 def create_crab(ds) :
232  dqmout = make_dqmname(ds)
233  hf = make_harv_name(ds)
234  out = open(f_crab, 'w')
235  out.write(crab_block)
236  out.write('\npset=' + hf)
237  out.write('datasetpath=' + ds)
238  out.write('\noutput_file=' + dqmout)
239  out.close()
240 
def make_harv_name(dset)
def create_crab(ds)
def make_dqmname(s)

◆ create_harvest()

def harvestRelVal.create_harvest (   ds)

Definition at line 156 of file harvestRelVal.py.

References find_dqmref(), get_cond_from_dsetpath(), make_harv_name(), and print().

156 def create_harvest(ds) :
157  raw_cmsdriver = "cmsDriver.py harvest -s HARVESTING:validationHarvesting --mc --conditions FrontierConditions_GlobalTag,STARTUP_30X::All --harvesting AtJobEnd --no_exec -n -1"
158  cmsdriver = raw_cmsdriver
159  cond = get_cond_from_dsetpath(ds)
160  if cond == 0 :
161  print('unexpected problem with conditions')
162  sys.exit(50)
163  cmsdriver = cmsdriver.replace('30X',cond)
164  fin_name="harvest_HARVESTING_STARTUP.py"
165  if ds.find('IDEAL') != -1 :
166  cmsdriver = cmsdriver.replace('STARTUP','IDEAL')
167  fin_name = fin_name.replace('STARTUP','IDEAL')
168  if ds.find('FastSim') != -1:
169  cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingFS')
170  if ds.find('PileUp') != -1:
171  cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingPU')
172 
173  #print "=>", cmsdriver, " fs?", ds.find('FastSim')
174  if os.path.exists(fin_name) :
175  os.system("rm " + fin_name)
176  print("executing cmsdriver command:\n\t", cmsdriver)
177  os.system(cmsdriver)
178  if not os.path.exists(fin_name) :
179  print('problem with cmsdriver file name')
180  sys.exit(40)
181  os.system("touch " + fin_name)
182  hf = make_harv_name(ds)
183  os.system('mv ' + fin_name + " " + hf)
184  out = open(hf, 'a')
185  out.write("\n\n##additions to cmsDriver output \n")
186  out.write("process.dqmSaver.workflow = '" + ds + "'\n")
187  if is_dbs :
188  out.write("process.source.fileNames = cms.untracked.vstring(\n")
189  for afile in api.listFiles(path=ds):
190  out.write(" '%s',\n" % afile['LogicalFileName'])
191  out.write(")\n")
192 
193  dqmref = find_dqmref(ds);
194  if not dqmref == 'NONE' :
195  out.write("process.dqmSaver.referenceHandling = 'all'\n")
196 
197  out.close()
198 
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def make_harv_name(dset)
def get_cond_from_dsetpath(ds)
def find_dqmref(ds)
def create_harvest(ds)

◆ create_mcrab()

def harvestRelVal.create_mcrab (   set,
  fcrab,
  fout 
)

Definition at line 199 of file harvestRelVal.py.

References append_sample_mcrab().

199 def create_mcrab(set, fcrab, fout):
200  out = open(fout, 'w')
201  out.write('[MULTICRAB]')
202  out.write('\ncfg=' + fcrab)
203  out.write('\n\n[COMMON]')
204  nevt = -1
205  njob = 1
206  out.write('\nCMSSW.total_number_of_events=' + (str)(nevt) )
207  out.write('\nCMSSW.number_of_jobs=' + (str)(njob) )
208  for s in set:
209  append_sample_mcrab(s, out)
210  out.close()
211 
def create_mcrab(set, fcrab, fout)
def append_sample_mcrab(dsetp, fout)

◆ find_dqmref()

def harvestRelVal.find_dqmref (   ds)

Definition at line 122 of file harvestRelVal.py.

References make_dqmname(), print(), python.rootplot.root2matplotlib.replace(), and str.

Referenced by append_sample_mcrab(), and create_harvest().

122 def find_dqmref(ds) :
123  if not do_reference :
124  return 'NONE'
125  cp = cmssw_ver[-1:]
126  ip = (int)(cp) - 1
127  ref_ver = cmssw_ver.replace(cp,str(ip))
128  #print "cms:", cmssw_ver, " cp:", cp, " ip:", ip, " new_ver:", ref_ver
129  ref_dir = "/castor/cern.ch/user/n/nuno/relval/harvest/" + ref_ver + "/"
130  ref_dsf = make_dqmname(ds.replace(cmssw_ver, ref_ver))
131  gls = " | grep root | grep "
132  #to accept crab appended _1.root in file names, nd skip versions/conditions
133  gls += ref_dsf[:-25]
134  gls += "| awk '{print $9}' "
135  #print "refds:", ref_dsf, " command: rfdir", ref_dir+gls
136  command = "rfcp " + ref_dir + "`rfdir " + ref_dir + gls + "` ."
137  #print "command:", command
138  os.system(command)
139  tmpfile = "ref.txt"
140  command = "ls -rtl *" + gls + " > " + tmpfile
141  #print "command:", command
142  os.system(command)
143  the_ref = 'NONE'
144  if os.path.exists(tmpfile) :
145  fin = open(tmpfile,'r')
146  ref = fin.readline().replace('\n','')
147  #print "read ref:", ref, "exists?", os.path.exists(ref)
148  fin.close()
149  if os.path.exists(ref) :
150  the_ref = ref
151  else :
152  the_ref = 'NONE'
153  print("Found reference file:", the_ref)
154  return the_ref
155 
def replace(string, replacements)
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def find_dqmref(ds)
#define str(s)
def make_dqmname(s)

◆ get_cond_from_dsetpath()

def harvestRelVal.get_cond_from_dsetpath (   ds)

Definition at line 48 of file harvestRelVal.py.

References spr.find(), print(), and python.rootplot.root2matplotlib.replace().

Referenced by check_dset(), and create_harvest().

48 def get_cond_from_dsetpath(ds) :
49  ca = ds.split('/')[2].replace(cmssw_ver+'_','').replace('IDEAL_','').replace('STARTUP_','').replace('_FastSim','')
50  cb = ca[:ca.find('v')-1]
51  if cb[0].find('3') == -1 or len(cb) > 3:
52  print("problem extracting condition for", ds, " : ", cb, '(len:',len(cb),')')
53  if cb.find('31X') != -1:
54  cb = '31X'
55  elif cb.find('30X') != -1:
56  cb = '30X'
57  else:
58  print("skipping", cb)
59  return 0
60  print("condition found:", cb)
61  else :
62  print("good condition for", ds, " : ", cb, '(len:',len(cb),')')
63  return cb
64 
65 
def replace(string, replacements)
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:19
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def get_cond_from_dsetpath(ds)

◆ get_name_from_dsetpath()

def harvestRelVal.get_name_from_dsetpath (   ds)

Definition at line 43 of file harvestRelVal.py.

References python.rootplot.root2matplotlib.replace().

Referenced by append_sample_mcrab(), and make_harv_name().

44  fs = ds.split('/')
45  fa = fs[1].replace('RelVal','')
46  return fa
47 
def get_name_from_dsetpath(ds)
def replace(string, replacements)

◆ make_dbs_list()

def harvestRelVal.make_dbs_list (   dbslf)

Definition at line 66 of file harvestRelVal.py.

References print().

66 def make_dbs_list(dbslf) :
67  if not is_dbs :
68  return
69  flis = open(dbslf,'w')
70  for ads in api.listDatasetPaths() :
71  if ads.find('RelVal') != -1 \
72  or ads.find(cmssw_ver) != -1 \
73  or ads.find("/GEN-SIM") != -1 :
74 # and ads.find("/GEN-SIM-RECO") != -1 :
75  flis.write(ads + '\n')
76  flis.close()
77  print('Generated dataset list', dbslf, 'from dbs.')
78  #exampe:
79  #dbs lsd --path=/RelVal*/CMSSW_3_1_0_pre5*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet > mylist.txt
80  #dbslsd = "dbs lsd --path=/RelVal*/" + cmssw_ver + "*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
81  #os.system( '`' + dbslsd + ' > ' + dbslf + '`')
82 
def make_dbs_list(dbslf)
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

◆ make_dqmname()

def harvestRelVal.make_dqmname (   s)

Definition at line 40 of file harvestRelVal.py.

Referenced by append_sample_mcrab(), create_crab(), and find_dqmref().

40 def make_dqmname(s):
41  return 'DQM_V0001_R000000001' + s.replace('/','__') + '.root'
42 
def make_dqmname(s)

◆ make_harv_name()

def harvestRelVal.make_harv_name (   dset)

Definition at line 212 of file harvestRelVal.py.

References get_name_from_dsetpath().

Referenced by append_sample_mcrab(), create_crab(), and create_harvest().

212 def make_harv_name(dset) :
213  return 'harvest_' + get_name_from_dsetpath(dset) + '.py'
214 
def get_name_from_dsetpath(ds)
def make_harv_name(dset)

◆ print_def()

def harvestRelVal.print_def ( )

Definition at line 18 of file harvestRelVal.py.

References print().

18 def print_def():
19  print("Usage:", sys.argv[0], "[list_of_processes]")
20  print("Examples:")
21  print("harvestRelVal.py")
22  print("harvestRelVal.py /RelValTTbar/CMSSW_3_1_0_pre4_STARTUP_30X_v1/GEN-SIM-RECO")
23  print("harvestRelVal.py <dataset_list.txt>")
24 
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

◆ read_ds_file()

def harvestRelVal.read_ds_file ( )

Definition at line 83 of file harvestRelVal.py.

References print().

83 def read_ds_file() :
84  if not os.path.exists(dsfile) :
85  print("problem reading file", dsfile)
86  sys.exit(30)
87  fin = open(dsfile,'r')
88  for dset in fin.readlines():
89  d = dset.replace('\n','')
90  if d.find('#') == -1 :
91  dsetpaths.append(d)
92  else :
93  print('skipping:', d)
94  fin.close()
95  print('Using data set list in ', dsfile)
96 
def read_ds_file()
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

Variable Documentation

◆ api

harvestRelVal.api

Definition at line 307 of file harvestRelVal.py.

◆ argin

harvestRelVal.argin

Definition at line 269 of file harvestRelVal.py.

◆ args

harvestRelVal.args

Definition at line 303 of file harvestRelVal.py.

◆ cmssw_ver

harvestRelVal.cmssw_ver

Definition at line 310 of file harvestRelVal.py.

◆ crab_block

harvestRelVal.crab_block

Definition at line 241 of file harvestRelVal.py.

◆ do_reference

harvestRelVal.do_reference

Definition at line 271 of file harvestRelVal.py.

◆ dsetpaths

harvestRelVal.dsetpaths

Definition at line 326 of file harvestRelVal.py.

◆ dsfile

harvestRelVal.dsfile

Definition at line 270 of file harvestRelVal.py.

◆ dslproc

harvestRelVal.dslproc

Definition at line 348 of file harvestRelVal.py.

◆ f_crab

harvestRelVal.f_crab

Create harvest.py template.

Create crab.cfg template

Definition at line 358 of file harvestRelVal.py.

◆ f_multi_crab

harvestRelVal.f_multi_crab

Create harvest_n.py for individual datasets.

Create multicrab.cfg

Definition at line 366 of file harvestRelVal.py.

◆ harvfilelist

harvestRelVal.harvfilelist

Print what has been created.

Definition at line 371 of file harvestRelVal.py.

◆ input_type

harvestRelVal.input_type

Definition at line 268 of file harvestRelVal.py.

◆ is_dbs

harvestRelVal.is_dbs

Definition at line 293 of file harvestRelVal.py.

◆ optManager

harvestRelVal.optManager

Definition at line 302 of file harvestRelVal.py.

◆ opts

harvestRelVal.opts

Definition at line 303 of file harvestRelVal.py.