CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
harvestRelVal Namespace Reference

Functions

def append_sample_mcrab
 
def check_dbs
 
def check_dset
 
def check_nevts_dset
 
def create_crab
 
def create_harvest
 
def create_mcrab
 
def find_dqmref
 
def get_cond_from_dsetpath
 
def get_name_from_dsetpath
 
def make_dbs_list
 
def make_dqmname
 
def make_harv_name
 
def print_def
 
def read_ds_file
 

Variables

tuple api = DbsApi(args)
 
string argin = ''
 
dictionary args = {}
 
tuple cmssw_ver = os.getenv('CMSSW_VERSION','NOTSET')
 
string crab_block
 
 do_reference = False
 
tuple dsetpaths = list()
 
string dsfile = ''
 
tuple dslproc = open("dset_processed.txt", 'w')
 
string f_crab = 'crab.cfg'
 Create harvest.py template. More...
 
string f_multi_crab = 'multicrab.cfg'
 Create harvest_n.py for individual datasets. More...
 
tuple harvfilelist = list()
 Print what has been created. More...
 
string input_type = ''
 
tuple is_dbs = check_dbs()
 
tuple optManager = DbsOptionParser()
 

Function Documentation

def harvestRelVal.append_sample_mcrab (   dsetp,
  fout 
)

Definition at line 215 of file harvestRelVal.py.

References find_dqmref(), get_name_from_dsetpath(), make_dqmname(), and make_harv_name().

Referenced by create_mcrab().

215 
216 def append_sample_mcrab(dsetp, fout):
217  dqm = make_dqmname(dsetp)
218  sample = get_name_from_dsetpath(dsetp)
219  hf = make_harv_name(dsetp)
220  if not os.path.exists(hf) :
221  print 'problem creating multicrab, file', hf, 'does not exist'
222  sys.exit(17)
223  fout.write('\n\n[' + sample + ']')
224  fout.write('\nCMSSW.pset=' + hf)
225  fout.write('\nCMSSW.datasetpath=' + dsetp)
226  fout.write('\nCMSSW.output_file=' + dqm)
227 
228  dqmref = find_dqmref(dsetp);
229  if not dqmref == 'NONE' :
230  fout.write('\nUSER.additional_input_files=' + dqmref)
harvestRelVal.get_name_from_dsetpath
def get_name_from_dsetpath
Definition: harvestRelVal.py:42
def harvestRelVal.check_dbs ( )

Definition at line 24 of file harvestRelVal.py.

24 
25 def check_dbs():
26  if os.getenv('DBSCMD_HOME','NOTSET') == 'NOTSET' :
27  return 0
28  return 1
def harvestRelVal.check_dset ( )

Definition at line 96 of file harvestRelVal.py.

References check_nevts_dset(), get_cond_from_dsetpath(), and list().

96 
97 def check_dset() :
98  #check cmssw consistency
99  for s in dsetpaths:
100  if s.find(cmssw_ver) == -1 :
101  dsetpaths.remove(s)
102  print 'Inconsistency found with datset and cmssw version (', cmssw_ver, ')' \
103  ': \t ', s, ' has been removed.'
104  #check conditions from dsetname
105  for s in dsetpaths[:]: #nb:need to make a copy here!
106  cond = get_cond_from_dsetpath(s)
107  if cond == 0 :
108  dsetpaths.remove(s)
109  #check list size
110  nSamples = len(dsetpaths)
111  if nSamples == 0 :
112  print "Empty input list, exit."
113  sys.exit(12)
114  else :
115  print 'Processing', nSamples, 'data sets.'
116  #check event numbers
117  nSampleEvts = list()
118  for s in dsetpaths:
119  nSampleEvts.append(check_nevts_dset(s))
120  print 'number of events per dataset:', nSampleEvts
harvestRelVal.get_cond_from_dsetpath
def get_cond_from_dsetpath
Definition: harvestRelVal.py:47
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def harvestRelVal.check_nevts_dset (   dset)

Definition at line 29 of file harvestRelVal.py.

Referenced by check_dset().

29 
30 def check_nevts_dset(dset):
31  if not is_dbs :
32  return -1
33  ntot=0
34  for afile in api.listFiles(path=str(dset)):
35  nevts = afile['NumberOfEvents']
36  ntot += nevts
37  #print " %s" % afile['LogicalFileName']
38  return ntot
def harvestRelVal.create_crab (   ds)

Definition at line 231 of file harvestRelVal.py.

References make_dqmname(), and make_harv_name().

231 
232 def create_crab(ds) :
233  dqmout = make_dqmname(ds)
234  hf = make_harv_name(ds)
235  out = open(f_crab, 'w')
236  out.write(crab_block)
237  out.write('\npset=' + hf)
238  out.write('datasetpath=' + ds)
239  out.write('\noutput_file=' + dqmout)
240  out.close()
def harvestRelVal.create_harvest (   ds)

Definition at line 155 of file harvestRelVal.py.

References find_dqmref(), get_cond_from_dsetpath(), and make_harv_name().

155 
156 def create_harvest(ds) :
157  raw_cmsdriver = "cmsDriver.py harvest -s HARVESTING:validationHarvesting --mc --conditions FrontierConditions_GlobalTag,STARTUP_30X::All --harvesting AtJobEnd --no_exec -n -1"
158  cmsdriver = raw_cmsdriver
159  cond = get_cond_from_dsetpath(ds)
160  if cond == 0 :
161  print 'unexpected problem with conditions'
162  sys.exit(50)
163  cmsdriver = cmsdriver.replace('30X',cond)
164  fin_name="harvest_HARVESTING_STARTUP.py"
165  if ds.find('IDEAL') != -1 :
166  cmsdriver = cmsdriver.replace('STARTUP','IDEAL')
167  fin_name = fin_name.replace('STARTUP','IDEAL')
168  if ds.find('FastSim') != -1:
169  cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingFS')
170  if ds.find('PileUp') != -1:
171  cmsdriver = cmsdriver.replace('validationHarvesting','validationHarvestingPU')
172 
173  #print "=>", cmsdriver, " fs?", ds.find('FastSim')
174  if os.path.exists(fin_name) :
175  os.system("rm " + fin_name)
176  print "executing cmsdriver command:\n\t", cmsdriver
177  os.system(cmsdriver)
178  if not os.path.exists(fin_name) :
179  print 'problem with cmsdriver file name'
180  sys.exit(40)
181  os.system("touch " + fin_name)
182  hf = make_harv_name(ds)
183  os.system('mv ' + fin_name + " " + hf)
184  out = open(hf, 'a')
185  out.write("\n\n##additions to cmsDriver output \n")
186  out.write("process.dqmSaver.workflow = '" + ds + "'\n")
187  if is_dbs :
188  out.write("process.source.fileNames = cms.untracked.vstring(\n")
189  for afile in api.listFiles(path=ds):
190  out.write(" '%s',\n" % afile['LogicalFileName'])
191  out.write(")\n")
192 
193  dqmref = find_dqmref(ds);
194  if not dqmref == 'NONE' :
195  out.write("process.DQMStore.referenceFileName = '" + dqmref + "'\n")
196  out.write("process.dqmSaver.referenceHandling = 'all'\n")
197 
198  out.close()
harvestRelVal.get_cond_from_dsetpath
def get_cond_from_dsetpath
Definition: harvestRelVal.py:47
def harvestRelVal.create_mcrab (   set,
  fcrab,
  fout 
)

Definition at line 199 of file harvestRelVal.py.

References append_sample_mcrab().

199 
200 def create_mcrab(set, fcrab, fout):
201  out = open(fout, 'w')
202  out.write('[MULTICRAB]')
203  out.write('\ncfg=' + fcrab)
204  out.write('\n\n[COMMON]')
205  nevt = -1
206  njob = 1
207  out.write('\nCMSSW.total_number_of_events=' + (str)(nevt) )
208  out.write('\nCMSSW.number_of_jobs=' + (str)(njob) )
209  for s in set:
210  append_sample_mcrab(s, out)
211  out.close()
def harvestRelVal.find_dqmref (   ds)

Definition at line 121 of file harvestRelVal.py.

References make_dqmname(), and python.rootplot.root2matplotlib.replace().

Referenced by append_sample_mcrab(), and create_harvest().

121 
122 def find_dqmref(ds) :
123  if not do_reference :
124  return 'NONE'
125  cp = cmssw_ver[-1:]
126  ip = (int)(cp) - 1
127  ref_ver = cmssw_ver.replace(cp,str(ip))
128  #print "cms:", cmssw_ver, " cp:", cp, " ip:", ip, " new_ver:", ref_ver
129  ref_dir = "/castor/cern.ch/user/n/nuno/relval/harvest/" + ref_ver + "/"
130  ref_dsf = make_dqmname(ds.replace(cmssw_ver, ref_ver))
131  gls = " | grep root | grep "
132  #to accept crab appended _1.root in file names, nd skip versions/conditions
133  gls += ref_dsf[:-25]
134  gls += "| awk '{print $9}' "
135  #print "refds:", ref_dsf, " command: rfdir", ref_dir+gls
136  command = "rfcp " + ref_dir + "`rfdir " + ref_dir + gls + "` ."
137  #print "command:", command
138  os.system(command)
139  tmpfile = "ref.txt"
140  command = "ls -rtl *" + gls + " > " + tmpfile
141  #print "command:", command
142  os.system(command)
143  the_ref = 'NONE'
144  if os.path.exists(tmpfile) :
145  fin = open(tmpfile,'r')
146  ref = fin.readline().replace('\n','')
147  #print "read ref:", ref, "exists?", os.path.exists(ref)
148  fin.close()
149  if os.path.exists(ref) :
150  the_ref = ref
151  else :
152  the_ref = 'NONE'
153  print "Found reference file:", the_ref
154  return the_ref
def harvestRelVal.get_cond_from_dsetpath (   ds)

Definition at line 47 of file harvestRelVal.py.

References spr.find(), and python.rootplot.root2matplotlib.replace().

Referenced by check_dset(), and create_harvest().

47 
48 def get_cond_from_dsetpath(ds) :
49  ca = ds.split('/')[2].replace(cmssw_ver+'_','').replace('IDEAL_','').replace('STARTUP_','').replace('_FastSim','')
50  cb = ca[:ca.find('v')-1]
51  if cb[0].find('3') == -1 or len(cb) > 3:
52  print "problem extracting condition for", ds, " : ", cb, '(len:',len(cb),')'
53  if cb.find('31X') != -1:
54  cb = '31X'
55  elif cb.find('30X') != -1:
56  cb = '30X'
57  else:
58  print "skipping", cb
59  return 0
60  print "condition found:", cb
61  else :
62  print "good condition for", ds, " : ", cb, '(len:',len(cb),')'
63  return cb
64 
spr::find
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:7
harvestRelVal.get_cond_from_dsetpath
def get_cond_from_dsetpath
Definition: harvestRelVal.py:47
def harvestRelVal.get_name_from_dsetpath (   ds)

Definition at line 42 of file harvestRelVal.py.

References python.rootplot.root2matplotlib.replace().

Referenced by append_sample_mcrab(), and make_harv_name().

42 
43 def get_name_from_dsetpath(ds):
44  fs = ds.split('/')
45  fa = fs[1].replace('RelVal','')
46  return fa
harvestRelVal.get_name_from_dsetpath
def get_name_from_dsetpath
Definition: harvestRelVal.py:42
def harvestRelVal.make_dbs_list (   dbslf)

Definition at line 65 of file harvestRelVal.py.

65 
66 def make_dbs_list(dbslf) :
67  if not is_dbs :
68  return
69  flis = open(dbslf,'w')
70  for ads in api.listDatasetPaths() :
71  if ads.find('RelVal') != -1 \
72  or ads.find(cmssw_ver) != -1 \
73  or ads.find("/GEN-SIM") != -1 :
74 # and ads.find("/GEN-SIM-RECO") != -1 :
75  flis.write(ads + '\n')
76  flis.close()
77  print 'Generated dataset list', dbslf, 'from dbs.'
78  #exampe:
79  #dbs lsd --path=/RelVal*/CMSSW_3_1_0_pre5*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet > mylist.txt
80  #dbslsd = "dbs lsd --path=/RelVal*/" + cmssw_ver + "*/GEN-SIM-RECO --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
81  #os.system( '`' + dbslsd + ' > ' + dbslf + '`')
def harvestRelVal.make_dqmname (   s)

Definition at line 39 of file harvestRelVal.py.

Referenced by append_sample_mcrab(), create_crab(), and find_dqmref().

39 
40 def make_dqmname(s):
41  return 'DQM_V0001_R000000001' + s.replace('/','__') + '.root'
def harvestRelVal.make_harv_name (   dset)

Definition at line 212 of file harvestRelVal.py.

References get_name_from_dsetpath().

Referenced by append_sample_mcrab(), create_crab(), and create_harvest().

212 
213 def make_harv_name(dset) :
214  return 'harvest_' + get_name_from_dsetpath(dset) + '.py'
harvestRelVal.get_name_from_dsetpath
def get_name_from_dsetpath
Definition: harvestRelVal.py:42
def harvestRelVal.print_def ( )

Definition at line 17 of file harvestRelVal.py.

17 
18 def print_def():
19  print "Usage:", sys.argv[0], "[list_of_processes]"
20  print "Examples:"
21  print "harvestRelVal.py"
22  print "harvestRelVal.py /RelValTTbar/CMSSW_3_1_0_pre4_STARTUP_30X_v1/GEN-SIM-RECO"
23  print "harvestRelVal.py <dataset_list.txt>"
def harvestRelVal.read_ds_file ( )

Definition at line 82 of file harvestRelVal.py.

82 
83 def read_ds_file() :
84  if not os.path.exists(dsfile) :
85  print "problem reading file", dsfile
86  sys.exit(30)
87  fin = open(dsfile,'r')
88  for dset in fin.readlines():
89  d = dset.replace('\n','')
90  if d.find('#') == -1 :
91  dsetpaths.append(d)
92  else :
93  print 'skipping:', d
94  fin.close()
95  print 'Using data set list in ', dsfile

Variable Documentation

tuple harvestRelVal.api = DbsApi(args)

Definition at line 307 of file harvestRelVal.py.

list harvestRelVal.argin = ''

Definition at line 269 of file harvestRelVal.py.

dictionary harvestRelVal.args = {}

Definition at line 305 of file harvestRelVal.py.

tuple harvestRelVal.cmssw_ver = os.getenv('CMSSW_VERSION','NOTSET')

Definition at line 310 of file harvestRelVal.py.

string harvestRelVal.crab_block
Initial value:
1 = """
2 [CRAB]
3 jobtype = cmssw
4 scheduler = glite
5 
6 [EDG]
7 remove_default_blacklist=1
8 rb = CERN
9 
10 [USER]
11 return_data = 1
12 #copy_data = 1
13 #storage_element=srm-cms.cern.ch
14 #storage_path=/srm/managerv2?SFN=/castor/cern.ch
15 #user_remote_dir=/user/n/nuno/test
16 publish_data=0
17 thresholdLevel=70
18 eMail=nuno@cern.ch
19 
20 [CMSSW]
21 total_number_of_events=-1
22 show_prod = 1
23 number_of_jobs=1
24 """

Definition at line 241 of file harvestRelVal.py.

harvestRelVal.do_reference = False

Definition at line 271 of file harvestRelVal.py.

tuple harvestRelVal.dsetpaths = list()

Definition at line 326 of file harvestRelVal.py.

string harvestRelVal.dsfile = ''

Definition at line 270 of file harvestRelVal.py.

tuple harvestRelVal.dslproc = open("dset_processed.txt", 'w')

Definition at line 348 of file harvestRelVal.py.

string harvestRelVal.f_crab = 'crab.cfg'

Create harvest.py template.

Create crab.cfg template

Definition at line 358 of file harvestRelVal.py.

string harvestRelVal.f_multi_crab = 'multicrab.cfg'

Create harvest_n.py for individual datasets.

Create multicrab.cfg

Definition at line 366 of file harvestRelVal.py.

tuple harvestRelVal.harvfilelist = list()

Print what has been created.

Definition at line 371 of file harvestRelVal.py.

string harvestRelVal.input_type = ''

Definition at line 268 of file harvestRelVal.py.

tuple harvestRelVal.is_dbs = check_dbs()

Definition at line 293 of file harvestRelVal.py.

tuple harvestRelVal.optManager = DbsOptionParser()

Definition at line 302 of file harvestRelVal.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5