CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
harvestRelVal Namespace Reference

Functions

def append_sample_mcrab
 
def check_dbs
 
def check_dset
 
def check_nevts_dset
 
def create_crab
 
def create_harvest
 
def create_mcrab
 
def find_dqmref
 
def get_cond_from_dsetpath
 
def get_name_from_dsetpath
 
def make_dbs_list
 
def make_dqmname
 
def make_harv_name
 
def print_def
 
def read_ds_file
 

Variables

tuple api = DbsApi(args)
 
string argin = ''
 
dictionary args = {}
 
tuple cmssw_ver = os.getenv('CMSSW_VERSION','NOTSET')
 
string crab_block
 
 do_reference = False
 
tuple dsetpaths = list()
 
string dsfile = ''
 
tuple dslproc = open("dset_processed.txt", 'w')
 
string f_crab = 'crab.cfg'
 Create harvest.py template. More...
 
string f_multi_crab = 'multicrab.cfg'
 Create harvest_n.py for individual datasets. More...
 
tuple harvfilelist = list()
 Print what has been created. More...
 
string input_type = ''
 
tuple is_dbs = check_dbs()
 
tuple optManager = DbsOptionParser()
 

Function Documentation

def harvestRelVal.append_sample_mcrab (   dsetp,
  fout 
)

Definition at line 215 of file harvestRelVal.py.

References find_dqmref(), get_name_from_dsetpath(), make_dqmname(), make_harv_name(), and print().

Referenced by create_mcrab().

216 def append_sample_mcrab(dsetp, fout):
217  dqm = make_dqmname(dsetp)
218  sample = get_name_from_dsetpath(dsetp)
219  hf = make_harv_name(dsetp)
220  if not os.path.exists(hf) :
221  print('problem creating multicrab, file', hf, 'does not exist')
222  sys.exit(17)
223  fout.write('\n\n[' + sample + ']')
224  fout.write('\nCMSSW.pset=' + hf)
225  fout.write('\nCMSSW.datasetpath=' + dsetp)
226  fout.write('\nCMSSW.output_file=' + dqm)
227 
228  dqmref = find_dqmref(dsetp);
229  if not dqmref == 'NONE' :
230  fout.write('\nUSER.additional_input_files=' + dqmref)
def get_name_from_dsetpath
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def harvestRelVal.check_dbs ( )

Definition at line 25 of file harvestRelVal.py.

25 
26 def check_dbs():
27  if os.getenv('DBSCMD_HOME','NOTSET') == 'NOTSET' :
28  return 0
29  return 1
def harvestRelVal.check_dset ( )

Definition at line 97 of file harvestRelVal.py.

References check_nevts_dset(), get_cond_from_dsetpath(), and print().

97 
98 def check_dset() :
99  #check cmssw consistency
100  for s in dsetpaths:
101  if s.find(cmssw_ver) == -1 :
102  dsetpaths.remove(s)
103  print('Inconsistency found with datset and cmssw version (', cmssw_ver, ')' \
104  ': \t ', s, ' has been removed.')
105  #check conditions from dsetname
106  for s in dsetpaths[:]: #nb:need to make a copy here!
107  cond = get_cond_from_dsetpath(s)
108  if cond == 0 :
109  dsetpaths.remove(s)
110  #check list size
111  nSamples = len(dsetpaths)
112  if nSamples == 0 :
113  print("Empty input list, exit.")
114  sys.exit(12)
115  else :
116  print('Processing', nSamples, 'data sets.')
117  #check event numbers
118  nSampleEvts = list()
119  for s in dsetpaths:
120  nSampleEvts.append(check_nevts_dset(s))
121  print('number of events per dataset:', nSampleEvts)
def get_cond_from_dsetpath
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def harvestRelVal.check_nevts_dset (   dset)

Definition at line 30 of file harvestRelVal.py.

References str.

Referenced by check_dset().

30 
31 def check_nevts_dset(dset):
32  if not is_dbs :
33  return -1
34  ntot=0
35  for afile in api.listFiles(path=str(dset)):
36  nevts = afile['NumberOfEvents']
37  ntot += nevts
38  #print " %s" % afile['LogicalFileName']
39  return ntot
#define str(s)
def harvestRelVal.create_crab (   ds)

Definition at line 231 of file harvestRelVal.py.

References make_dqmname(), and make_harv_name().

232 def create_crab(ds) :
233  dqmout = make_dqmname(ds)
234  hf = make_harv_name(ds)
235  out = open(f_crab, 'w')
236  out.write(crab_block)
237  out.write('\npset=' + hf)
238  out.write('datasetpath=' + ds)
239  out.write('\noutput_file=' + dqmout)
240  out.close()
def harvestRelVal.create_harvest (   ds)

Definition at line 156 of file harvestRelVal.py.

References find_dqmref(), get_cond_from_dsetpath(), make_harv_name(), and print().

157 def create_harvest(ds) :
158  raw_cmsdriver = "cmsDriver.py harvest -s HARVESTING:validationHarvesting --mc --conditions FrontierConditions_GlobalTag,STARTUP_30X::All --harvesting AtJobEnd --no_exec -n -1"
159  cmsdriver = raw_cmsdriver
160  cond = get_cond_from_dsetpath(ds)
161  if cond == 0 :
162  print('unexpected problem with conditions')
163  sys.exit(50)
164  cmsdriver = cmsdriver.replace('30X',cond)
165  fin_name="harvest_HARVESTING_STARTUP.py"
166  if ds.find('IDEAL') != -1 :
167  cmsdriver = cmsdriver.replace('STARTUP','IDEAL')
168  fin_name = fin_name.replace('STARTUP','IDEAL')
169  if ds.find('FastSim') != -1:
170  cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingFS')
171  if ds.find('PileUp') != -1:
172  cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingPU')
173 
174  #print "=>", cmsdriver, " fs?", ds.find('FastSim')
175  if os.path.exists(fin_name) :
176  os.system("rm " + fin_name)
177  print("executing cmsdriver command:\n\t", cmsdriver)
178  os.system(cmsdriver)
179  if not os.path.exists(fin_name) :
180  print('problem with cmsdriver file name')
181  sys.exit(40)
182  os.system("touch " + fin_name)
183  hf = make_harv_name(ds)
184  os.system('mv ' + fin_name + " " + hf)
185  out = open(hf, 'a')
186  out.write("\n\n##additions to cmsDriver output \n")
187  out.write("process.dqmSaver.workflow = '" + ds + "'\n")
188  if is_dbs :
189  out.write("process.source.fileNames = cms.untracked.vstring(\n")
190  for afile in api.listFiles(path=ds):
191  out.write(" '%s',\n" % afile['LogicalFileName'])
192  out.write(")\n")
193 
194  dqmref = find_dqmref(ds);
195  if not dqmref == 'NONE' :
196  out.write("process.dqmSaver.referenceHandling = 'all'\n")
197 
198  out.close()
def get_cond_from_dsetpath
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def harvestRelVal.create_mcrab (   set,
  fcrab,
  fout 
)

Definition at line 199 of file harvestRelVal.py.

References append_sample_mcrab().

200 def create_mcrab(set, fcrab, fout):
201  out = open(fout, 'w')
202  out.write('[MULTICRAB]')
203  out.write('\ncfg=' + fcrab)
204  out.write('\n\n[COMMON]')
205  nevt = -1
206  njob = 1
207  out.write('\nCMSSW.total_number_of_events=' + (str)(nevt) )
208  out.write('\nCMSSW.number_of_jobs=' + (str)(njob) )
209  for s in set:
210  append_sample_mcrab(s, out)
211  out.close()
def harvestRelVal.find_dqmref (   ds)

Definition at line 122 of file harvestRelVal.py.

References make_dqmname(), print(), python.rootplot.root2matplotlib.replace(), and str.

Referenced by append_sample_mcrab(), and create_harvest().

123 def find_dqmref(ds) :
124  if not do_reference :
125  return 'NONE'
126  cp = cmssw_ver[-1:]
127  ip = (int)(cp) - 1
128  ref_ver = cmssw_ver.replace(cp,str(ip))
129  #print "cms:", cmssw_ver, " cp:", cp, " ip:", ip, " new_ver:", ref_ver
130  ref_dir = "/castor/cern.ch/user/n/nuno/relval/harvest/" + ref_ver + "/"
131  ref_dsf = make_dqmname(ds.replace(cmssw_ver, ref_ver))
132  gls = " | grep root | grep "
133  #to accept crab appended _1.root in file names, nd skip versions/conditions
134  gls += ref_dsf[:-25]
135  gls += "| awk '{print $9}' "
136  #print "refds:", ref_dsf, " command: rfdir", ref_dir+gls
137  command = "rfcp " + ref_dir + "`rfdir " + ref_dir + gls + "` ."
138  #print "command:", command
139  os.system(command)
140  tmpfile = "ref.txt"
141  command = "ls -rtl *" + gls + " > " + tmpfile
142  #print "command:", command
143  os.system(command)
144  the_ref = 'NONE'
145  if os.path.exists(tmpfile) :
146  fin = open(tmpfile,'r')
147  ref = fin.readline().replace('\n','')
148  #print "read ref:", ref, "exists?", os.path.exists(ref)
149  fin.close()
150  if os.path.exists(ref) :
151  the_ref = ref
152  else :
153  the_ref = 'NONE'
154  print("Found reference file:", the_ref)
155  return the_ref
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
#define str(s)
def harvestRelVal.get_cond_from_dsetpath (   ds)

Definition at line 48 of file harvestRelVal.py.

References spr.find(), print(), and python.rootplot.root2matplotlib.replace().

Referenced by check_dset(), and create_harvest().

48 
49 def get_cond_from_dsetpath(ds) :
50  ca = ds.split('/')[2].replace(cmssw_ver+'_','').replace('IDEAL_','').replace('STARTUP_','').replace('_FastSim','')
51  cb = ca[:ca.find('v')-1]
52  if cb[0].find('3') == -1 or len(cb) > 3:
53  print("problem extracting condition for", ds, " : ", cb, '(len:',len(cb),')')
54  if cb.find('31X') != -1:
55  cb = '31X'
56  elif cb.find('30X') != -1:
57  cb = '30X'
58  else:
59  print("skipping", cb)
60  return 0
61  print("condition found:", cb)
62  else :
63  print("good condition for", ds, " : ", cb, '(len:',len(cb),')')
64  return cb
65 
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:19
def get_cond_from_dsetpath
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def harvestRelVal.get_name_from_dsetpath (   ds)

Definition at line 43 of file harvestRelVal.py.

References python.rootplot.root2matplotlib.replace().

Referenced by append_sample_mcrab(), and make_harv_name().

43 
45  fs = ds.split('/')
46  fa = fs[1].replace('RelVal','')
47  return fa
def get_name_from_dsetpath
def harvestRelVal.make_dbs_list (   dbslf)

Definition at line 66 of file harvestRelVal.py.

References print().

66 
67 def make_dbs_list(dbslf) :
68  if not is_dbs :
69  return
70  flis = open(dbslf,'w')
71  for ads in api.listDatasetPaths() :
72  if ads.find('RelVal') != -1 \
73  or ads.find(cmssw_ver) != -1 \
74  or ads.find("/GEN-SIM") != -1 :
75 # and ads.find("/GEN-SIM-RECO") != -1 :
76  flis.write(ads + '\n')
77  flis.close()
78  print('Generated dataset list', dbslf, 'from dbs.')
79  #exampe:
80  #dbs lsd --path=/RelVal*/CMSSW_3_1_0_pre5*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet > mylist.txt
81  #dbslsd = "dbs lsd --path=/RelVal*/" + cmssw_ver + "*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
82  #os.system( '`' + dbslsd + ' > ' + dbslf + '`')
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def harvestRelVal.make_dqmname (   s)

Definition at line 40 of file harvestRelVal.py.

Referenced by append_sample_mcrab(), create_crab(), and find_dqmref().

40 
41 def make_dqmname(s):
42  return 'DQM_V0001_R000000001' + s.replace('/','__') + '.root'
def harvestRelVal.make_harv_name (   dset)

Definition at line 212 of file harvestRelVal.py.

References get_name_from_dsetpath().

Referenced by append_sample_mcrab(), create_crab(), and create_harvest().

213 def make_harv_name(dset) :
214  return 'harvest_' + get_name_from_dsetpath(dset) + '.py'
def get_name_from_dsetpath
def harvestRelVal.print_def ( )

Definition at line 18 of file harvestRelVal.py.

References print().

18 
19 def print_def():
20  print("Usage:", sys.argv[0], "[list_of_processes]")
21  print("Examples:")
22  print("harvestRelVal.py")
23  print("harvestRelVal.py /RelValTTbar/CMSSW_3_1_0_pre4_STARTUP_30X_v1/GEN-SIM-RECO")
24  print("harvestRelVal.py <dataset_list.txt>")
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def harvestRelVal.read_ds_file ( )

Definition at line 83 of file harvestRelVal.py.

References print().

83 
84 def read_ds_file() :
85  if not os.path.exists(dsfile) :
86  print("problem reading file", dsfile)
87  sys.exit(30)
88  fin = open(dsfile,'r')
89  for dset in fin.readlines():
90  d = dset.replace('\n','')
91  if d.find('#') == -1 :
92  dsetpaths.append(d)
93  else :
94  print('skipping:', d)
95  fin.close()
96  print('Using data set list in ', dsfile)
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

Variable Documentation

tuple harvestRelVal.api = DbsApi(args)

Definition at line 307 of file harvestRelVal.py.

list harvestRelVal.argin = ''

Definition at line 269 of file harvestRelVal.py.

dictionary harvestRelVal.args = {}

Definition at line 305 of file harvestRelVal.py.

tuple harvestRelVal.cmssw_ver = os.getenv('CMSSW_VERSION','NOTSET')

Definition at line 310 of file harvestRelVal.py.

string harvestRelVal.crab_block
Initial value:
1 = """
2 [CRAB]
3 jobtype = cmssw
4 scheduler = glite
5 
6 [EDG]
7 remove_default_blacklist=1
8 rb = CERN
9 
10 [USER]
11 return_data = 1
12 #copy_data = 1
13 #storage_element=srm-cms.cern.ch
14 #storage_path=/srm/managerv2?SFN=/castor/cern.ch
15 #user_remote_dir=/user/n/nuno/test
16 publish_data=0
17 thresholdLevel=70
18 eMail=nuno@cern.ch
19 
20 [CMSSW]
21 total_number_of_events=-1
22 show_prod = 1
23 number_of_jobs=1
24 """

Definition at line 241 of file harvestRelVal.py.

harvestRelVal.do_reference = False

Definition at line 271 of file harvestRelVal.py.

tuple harvestRelVal.dsetpaths = list()

Definition at line 326 of file harvestRelVal.py.

string harvestRelVal.dsfile = ''

Definition at line 270 of file harvestRelVal.py.

tuple harvestRelVal.dslproc = open("dset_processed.txt", 'w')

Definition at line 348 of file harvestRelVal.py.

string harvestRelVal.f_crab = 'crab.cfg'

Create harvest.py template.

Create crab.cfg template

Definition at line 358 of file harvestRelVal.py.

string harvestRelVal.f_multi_crab = 'multicrab.cfg'

Create harvest_n.py for individual datasets.

Create multicrab.cfg

Definition at line 366 of file harvestRelVal.py.

tuple harvestRelVal.harvfilelist = list()

Print what has been created.

Definition at line 371 of file harvestRelVal.py.

string harvestRelVal.input_type = ''

Definition at line 268 of file harvestRelVal.py.

tuple harvestRelVal.is_dbs = check_dbs()

Definition at line 293 of file harvestRelVal.py.

tuple harvestRelVal.optManager = DbsOptionParser()

Definition at line 302 of file harvestRelVal.py.