CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions
tools Namespace Reference

Functions

def addCrabInputFile
 
def copyFilesFromCastor
 
def copyFilesLocal
 
def dqmWorkflowName
 
def getDatasetStr
 
def haddInCastor
 
def haddLocal
 
def listFilesInCastor
 
def listFilesLocal
 
def loadCmsProcess
 
def loadCmsProcessFile
 
def loadCrabCfg
 
def loadCrabDefault
 
def parseInput
 
def prependPaths
 
def replaceTemplate
 
def setGridEnv
 
def writeCfg
 
def writeCfgPkl
 

Function Documentation

def tools.addCrabInputFile (   crabCfg,
  inputFile 
)

Definition at line 183 of file tools.py.

Referenced by DTDQMValidation.DTDQMValidation.initCrab(), DTAnalysisResiduals.DTAnalysisResiduals.initCrab(), DTTTrigValid.DTTTrigValid.initCrab(), DTResidualCalibration.DTResidualCalibration.initCrab(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initCrab(), and DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initCrab().

183 
184 def addCrabInputFile(crabCfg,inputFile):
185  additionalInputFiles = ''
186  if crabCfg.has_option('USER','additional_input_files'):
187  additionalInputFiles = crabCfg.get('USER','additional_input_files')
188 
189  if additionalInputFiles: additionalInputFiles += ',%s' % inputFile
190  else: additionalInputFiles = inputFile
191 
192  crabCfg.set('USER','additional_input_files',additionalInputFiles)
193 
194  return crabCfg
tools.addCrabInputFile
def addCrabInputFile
Definition: tools.py:183
def tools.copyFilesFromCastor (   castor_dir,
  output_dir,
  type = 'root' 
)

Definition at line 55 of file tools.py.

References listFilesInCastor().

55 
56 def copyFilesFromCastor(castor_dir,output_dir,type='root'):
57  from subprocess import call
58  files = listFilesInCastor(castor_dir,type,'')
59 
60  print "Copying files from %s to %s" % (castor_dir,output_dir)
61  for item in files:
62  cmd = ['rfcp',item,output_dir]
63  print "..." + item
64  retcode = call(cmd)
65  if retcode != 0: raise RuntimeError('Error in copying file %s to directory %s' % (item,output_dir))
66 
67  return 0
tools.listFilesInCastor
def listFilesInCastor
Definition: tools.py:30
tools.copyFilesFromCastor
def copyFilesFromCastor
Definition: tools.py:55
def tools.copyFilesLocal (   dir,
  output_dir,
  type = 'root' 
)

Definition at line 68 of file tools.py.

References listFilesLocal().

68 
69 def copyFilesLocal(dir,output_dir,type='root'):
70  if not dir: raise ValueError('Please specify valid dir')
71  if not output_dir: raise ValueError('Please specify valid output dir')
72 
73  from subprocess import call
74  files = listFilesLocal(dir,type)
75  cmd = ['cp']
76  cmd.extend(files)
77  cmd.append(output_dir)
78  print cmd
79  retcode = call(cmd)
80  return retcode
tools.listFilesLocal
def listFilesLocal
Definition: tools.py:42
tools.copyFilesLocal
def copyFilesLocal
Definition: tools.py:68
def tools.dqmWorkflowName (   datasetpath,
  type,
  rev = 1 
)

Definition at line 23 of file tools.py.

Referenced by DTDqm.DTDqm.initProcess().

23 
24 def dqmWorkflowName(datasetpath,type,rev=1):
25  workflowName = datasetpath
26  sections = workflowName.split('/')[1:]
27  workflowName = '/%s/%s-%s-rev%d/%s' % (sections[0],sections[1],type,rev,sections[2])
28 
29  return workflowName
tools.dqmWorkflowName
def dqmWorkflowName
Definition: tools.py:23
def tools.getDatasetStr (   datasetpath)

Definition at line 15 of file tools.py.

Referenced by DTDQMMerge.DTDQMMerge.initProcess(), DTCalibrationWorker.DTCalibrationWorker.runAnalysisResiduals(), DTCalibrationWorker.DTCalibrationWorker.runResidualCalib(), DTCalibrationWorker.DTCalibrationWorker.runTtrigProd(), DTCalibrationWorker.DTCalibrationWorker.runTtrigValid(), DTCalibrationWorker.DTCalibrationWorker.runValidationWorkflow(), DTCalibrationWorker.DTCalibrationWorker.runVDriftMeanTimerCalib(), and DTCalibrationWorker.DTCalibrationWorker.runVDriftSegmentCalib().

15 
16 def getDatasetStr(datasetpath):
17  datasetstr = datasetpath
18  datasetstr.strip()
19  if datasetstr[0] == '/': datasetstr = datasetstr[1:]
20  datasetstr = datasetstr.replace('/','_')
21 
22  return datasetstr
tools.getDatasetStr
def getDatasetStr
Definition: tools.py:15
def tools.haddInCastor (   castor_dir,
  result_file,
  type = 'root',
  prefix = 'rfio:',
  suffix = None 
)

Definition at line 81 of file tools.py.

References listFilesInCastor().

81 
82 def haddInCastor(castor_dir,result_file,type = 'root',prefix = 'rfio:',suffix = None):
83  if not castor_dir: raise ValueError('Please specify valid castor dir')
84  if not result_file: raise ValueError('Please specify valid output file name')
85 
86  #cmd = 'hadd %s `./listfilesCastor %s | grep %s`'%(result_file,castor_dir,type)
87  #print "Running",cmd
88  #os.system(cmd)
89  from subprocess import call
90  files = listFilesInCastor(castor_dir,type,prefix)
91  if suffix: files = [item + suffix for item in files]
92 
93  cmd = ['hadd',result_file]
94  cmd.extend(files)
95  #print cmd
96  retcode = call(cmd)
97  return retcode
tools.listFilesInCastor
def listFilesInCastor
Definition: tools.py:30
tools.haddInCastor
def haddInCastor
Definition: tools.py:81
def tools.haddLocal (   dir,
  result_file,
  type = 'root' 
)

Definition at line 98 of file tools.py.

References listFilesLocal().

98 
99 def haddLocal(dir,result_file,type = 'root'):
100  if not dir: raise ValueError('Please specify valid dir')
101  if not result_file: raise ValueError('Please specify valid output file name')
102 
103  from subprocess import call
104  files = listFilesLocal(dir,type)
105  cmd = ['hadd',result_file]
106  cmd.extend(files)
107  #print cmd
108  retcode = call(cmd)
109  return retcode
tools.listFilesLocal
def listFilesLocal
Definition: tools.py:42
tools.haddLocal
def haddLocal
Definition: tools.py:98
def tools.listFilesInCastor (   castor_dir,
  type = 'root',
  prefix = 'rfio:' 
)

Definition at line 30 of file tools.py.

Referenced by copyFilesFromCastor(), haddInCastor(), DTDqm.runDQM(), DTDQMMerge.runDQM(), and DTCalibrationWorker.DTCalibrationWorker.runDQMHarvesting().

30 
31 def listFilesInCastor(castor_dir,type = 'root',prefix = 'rfio:'):
32  if not castor_dir: raise ValueError('Please specify valid castor dir')
33 
34  from subprocess import Popen,PIPE
35  p1 = Popen(['nsls',castor_dir],stdout=PIPE)
36  #p2 = Popen(['grep',type],stdin=p1.stdout,stdout=PIPE)
37  #files = [prefix + castor_dir + "/" + item[:-1] for item in p2.stdout]
38  #p2.stdout.close()
39  files = [ "%s%s/%s" % (prefix,castor_dir,item.rstrip()) for item in p1.stdout if item.find(type) != -1 ]
40  p1.stdout.close()
41  return files
tools.listFilesInCastor
def listFilesInCastor
Definition: tools.py:30
def tools.listFilesLocal (   dir,
  type = 'root' 
)

Definition at line 42 of file tools.py.

Referenced by copyFilesLocal(), haddLocal(), DTCalibrationWorker.DTCalibrationWorker.runDQMHarvesting(), and DTCalibrationWorker.DTCalibrationWorker.runDQMMerge().

42 
43 def listFilesLocal(dir,type = 'root'):
44  if not dir: raise ValueError('Please specify valid dir')
45 
46  #from subprocess import Popen,PIPE
47  #p1 = Popen(['ls',dir],stdout=PIPE)
48  #p2 = Popen(['grep',type],stdin=p1.stdout,stdout=PIPE)
49  #files = [dir + "/" + item[:-1] for item in p2.stdout]
50  #p2.stdout.close()
51  files = os.listdir(dir)
52  files = [ "%s/%s" % (dir,item) for item in files if item.find(type) != -1 ]
53 
54  return files
tools.listFilesLocal
def listFilesLocal
Definition: tools.py:42
def tools.loadCmsProcess (   psetPath)

Definition at line 144 of file tools.py.

Referenced by DTValidSummary.DTValidSummary.initProcess(), DTDqm.DTDqm.initProcess(), DTDQMMerge.DTDQMMerge.initProcess(), DTAnalysisResiduals.DTAnalysisResiduals.initProcess(), DTTTrigProd.DTTTrigProd.initProcess(), DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initProcess(), DTDQMHarvesting.DTDQMHarvesting.initProcess(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initProcess(), DTNoiseCalibration.DTNoiseCalibration.initProcess(), DTResidualCalibration.DTResidualCalibration.initProcess(), DTDQMValidation.DTDQMValidation.initProcess(), and DTTTrigValid.DTTTrigValid.initProcess().

144 
145 def loadCmsProcess(psetPath):
146  module = __import__(psetPath)
147  process = sys.modules[psetPath].process
148 
149  import copy
150  #FIXME: clone process
151  #processNew = copy.deepcopy(process)
152  processNew = copy.copy(process)
153  return processNew
tools.loadCmsProcess
def loadCmsProcess
Definition: tools.py:144
def tools.loadCmsProcessFile (   psetName)

Definition at line 140 of file tools.py.

140 
141 def loadCmsProcessFile(psetName):
142  pset = imp.load_source("psetmodule",psetName)
143  return pset.process
tools.loadCmsProcessFile
def loadCmsProcessFile
Definition: tools.py:140
def tools.loadCrabCfg (   cfgName = None)

Definition at line 178 of file tools.py.

Referenced by DTNoiseCalibration.DTNoiseCalibration.initCrab(), DTTTrigProd.DTTTrigProd.initCrab(), DTDQMValidation.DTDQMValidation.initCrab(), DTResidualCalibration.DTResidualCalibration.initCrab(), DTAnalysisResiduals.DTAnalysisResiduals.initCrab(), DTTTrigValid.DTTTrigValid.initCrab(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initCrab(), and DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initCrab().

178 
179 def loadCrabCfg(cfgName=None):
180  config = ConfigParser.ConfigParser()
181  if cfgName: config.read(cfgName)
182  return config
tools.loadCrabCfg
def loadCrabCfg
Definition: tools.py:178
def tools.loadCrabDefault (   crabCfg,
  config 
)

Definition at line 195 of file tools.py.

Referenced by DTNoiseCalibration.DTNoiseCalibration.initCrab(), DTTTrigProd.DTTTrigProd.initCrab(), DTDQMValidation.DTDQMValidation.initCrab(), DTResidualCalibration.DTResidualCalibration.initCrab(), DTAnalysisResiduals.DTAnalysisResiduals.initCrab(), DTTTrigValid.DTTTrigValid.initCrab(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initCrab(), and DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initCrab().

195 
196 def loadCrabDefault(crabCfg,config):
197  # CRAB section
198  if not crabCfg.has_section('CRAB'): crabCfg.add_section('CRAB')
199  crabCfg.set('CRAB','jobtype','cmssw')
200 
201  if hasattr(config,'scheduler') and config.scheduler: crabCfg.set('CRAB','scheduler',config.scheduler)
202  else: crabCfg.set('CRAB','scheduler','CAF')
203 
204  if hasattr(config,'useserver') and config.useserver: crabCfg.set('CRAB','use_server',1)
205 
206  # CMSSW section
207  if not crabCfg.has_section('CMSSW'): crabCfg.add_section('CMSSW')
208  if hasattr(config,'datasetpath') and config.datasetpath: crabCfg.set('CMSSW','datasetpath',config.datasetpath)
209  else: crabCfg.set('CMSSW','datasetpath','/XXX/YYY/ZZZ')
210  crabCfg.set('CMSSW','pset','pset.py')
211 
212  # Splitting config
213  crabCfg.remove_option('CMSSW','total_number_of_events')
214  crabCfg.remove_option('CMSSW','events_per_job')
215  crabCfg.remove_option('CMSSW','number_of_jobs')
216  crabCfg.remove_option('CMSSW','total_number_of_lumis')
217  crabCfg.remove_option('CMSSW','lumis_per_job')
218  crabCfg.remove_option('CMSSW','lumi_mask')
219  crabCfg.remove_option('CMSSW','split_by_run')
220 
221  """
222  if hasattr(config,'totalnumberevents'): crabCfg.set('CMSSW','total_number_of_events',config.totalnumberevents)
223  if hasattr(config,'eventsperjob'): crabCfg.set('CMSSW','events_per_job',config.eventsperjob)
224  """
225  if hasattr(config,'splitByLumi') and config.splitByLumi:
226  crabCfg.set('CMSSW','total_number_of_lumis',config.totalnumberlumis)
227  crabCfg.set('CMSSW','lumis_per_job',config.lumisperjob)
228  if hasattr(config,'lumimask') and config.lumimask: crabCfg.set('CMSSW','lumi_mask',config.lumimask)
229  elif hasattr(config,'splitByEvent') and config.splitByEvent:
230  crabCfg.set('CMSSW','total_number_of_events',config.totalnumberevents)
231  crabCfg.set('CMSSW','events_per_job',config.eventsperjob)
232  else:
233  crabCfg.set('CMSSW','split_by_run',1)
234 
235  if hasattr(config,'splitByEvent') and config.splitByEvent:
236  crabCfg.remove_option('CMSSW','runselection')
237  else:
238  if hasattr(config,'runselection') and config.runselection:
239  crabCfg.set('CMSSW','runselection',config.runselection)
240 
241  # USER section
242  if not crabCfg.has_section('USER'): crabCfg.add_section('USER')
243 
244  # Stageout config
245  if hasattr(config,'stageOutCAF') and config.stageOutCAF:
246  crabCfg.set('USER','return_data',0)
247  crabCfg.set('USER','copy_data',1)
248  crabCfg.set('USER','storage_element','T2_CH_CAF')
249  crabCfg.set('USER','user_remote_dir',config.userdircaf)
250  crabCfg.set('USER','check_user_remote_dir',0)
251  elif hasattr(config,'stageOutLocal') and config.stageOutLocal:
252  crabCfg.set('USER','return_data',1)
253  crabCfg.set('USER','copy_data',0)
254  crabCfg.remove_option('USER','storage_element')
255  crabCfg.remove_option('USER','user_remote_dir')
256  crabCfg.remove_option('USER','check_user_remote_dir')
257 
258  if hasattr(config,'email') and config.email: crabCfg.set('USER','eMail',config.email)
259  crabCfg.set('USER','xml_report','crabReport.xml')
260 
261  if hasattr(config,'runOnGrid') and config.runOnGrid:
262  crabCfg.remove_section('CAF')
263  if hasattr(config,'ce_black_list'):
264  if not crabCfg.has_section('GRID'): crabCfg.add_section('GRID')
265  crabCfg.set('GRID','ce_black_list', config.ce_black_list)
266  if hasattr(config,'ce_white_list'):
267  if not crabCfg.has_section('GRID'): crabCfg.add_section('GRID')
268  crabCfg.set('GRID','ce_white_list', config.ce_white_list)
269  else:
270  if not crabCfg.has_section('CAF'): crabCfg.add_section('CAF')
271  crabCfg.set('CAF','queue',config.queueAtCAF)
272 
273  return crabCfg
tools.loadCrabDefault
def loadCrabDefault
Definition: tools.py:195
def tools.parseInput (   inputFields,
  requiredFields = () 
)

Definition at line 122 of file tools.py.

122 
123 def parseInput(inputFields,requiredFields = ()):
124 
125  class options: pass
126  for item in sys.argv:
127  option = item.split('=')[0]
128  if option in inputFields:
129  value = item.split('=')[1]
130  if value in ('true','True','yes','Yes'): value = True
131  elif value in ('false','False','no','No'): value = False
132 
133  setattr(options,option,value)
134 
135  for item in requiredFields:
136  if not hasattr(options,item):
137  raise RuntimeError('Need to set "%s"' % item)
138 
139  return options
tools.parseInput
def parseInput
Definition: tools.py:122
def tools.prependPaths (   process,
  seqname 
)

Definition at line 154 of file tools.py.

154 
155 def prependPaths(process,seqname):
156  for path in process.paths:
157  getattr(process,path)._seq = getattr(process,seqname)*getattr(process,path)._seq
tools.prependPaths
def prependPaths
Definition: tools.py:154
def tools.replaceTemplate (   template,
  opts 
)

Definition at line 5 of file tools.py.

References SiPixelLorentzAngle_cfi.read.

5 
6 def replaceTemplate(template,**opts):
7  result = open(template).read()
8  for item in opts:
9  old = '@@%s@@'%item
10  new = str(opts[item])
11  print "Replacing",old,"to",new
12  result = result.replace(old,new)
13 
14  return result
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7
tools.replaceTemplate
def replaceTemplate
Definition: tools.py:5
def tools.setGridEnv (   cmssw_dir)

Definition at line 110 of file tools.py.

110 
111 def setGridEnv(cmssw_dir):
112  cwd = os.getcwd()
113  os.chdir(cmssw_dir)
114 
115  os.system('source /afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh')
116  os.system('cmsenv')
117  os.system('source /afs/cern.ch/cms/ccs/wm/scripts/Crab/crab.sh')
118 
119  os.chdir(cwd)
120 
121  return
tools.setGridEnv
def setGridEnv
Definition: tools.py:110
def tools.writeCfg (   process,
  dir,
  psetName 
)

Definition at line 158 of file tools.py.

References TablePrint.write.

158 
159 def writeCfg(process,dir,psetName):
160  if not os.path.exists(dir): os.makedirs(dir)
161  open(dir + '/' + psetName,'w').write(process.dumpPython())
tools.writeCfg
def writeCfg
Definition: tools.py:158
def tools.writeCfgPkl (   process,
  dir,
  psetName 
)

Definition at line 162 of file tools.py.

162 
163 def writeCfgPkl(process,dir,psetName):
164  if not os.path.exists(dir): os.makedirs(dir)
165 
166  pklFileName = psetName.split('.')[0] + '.pkl'
167  pklFile = open(dir + '/' + pklFileName,"wb")
168  myPickle = pickle.Pickler(pklFile)
169  myPickle.dump(process)
170  pklFile.close()
171 
172  outFile = open(dir + '/' + psetName,"w")
173  outFile.write("import FWCore.ParameterSet.Config as cms\n")
174  outFile.write("import pickle\n")
175  outFile.write("process = pickle.load(open('%s', 'rb'))\n" % pklFileName)
176  outFile.close()
177 
tools.writeCfgPkl
def writeCfgPkl
Definition: tools.py:162
Generated for CMSSW Reference Manual by doxygen 1.8.5