CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions
tools Namespace Reference

Functions

def addCrabInputFile
 
def copyFilesFromCastor
 
def copyFilesLocal
 
def create_single_iov_db
 
def dqmWorkflowName
 
def getDatasetStr
 
def haddInCastor
 
def haddLocal
 
def listFilesInCastor
 
def listFilesLocal
 
def loadCmsProcess
 
def loadCmsProcessFile
 
def loadCrabCfg
 
def loadCrabDefault
 
def parseInput
 
def prependPaths
 
def replaceTemplate
 
def run_checked
 
def setGridEnv
 
def writeCfg
 
def writeCfgPkl
 

Function Documentation

def tools.addCrabInputFile (   crabCfg,
  inputFile 
)

Definition at line 183 of file tools.py.

Referenced by DTDQMValidation.DTDQMValidation.initCrab(), DTAnalysisResiduals.DTAnalysisResiduals.initCrab(), DTTTrigValid.DTTTrigValid.initCrab(), DTResidualCalibration.DTResidualCalibration.initCrab(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initCrab(), and DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initCrab().

183 
184 def addCrabInputFile(crabCfg,inputFile):
185  additionalInputFiles = ''
186  if crabCfg.has_option('USER','additional_input_files'):
187  additionalInputFiles = crabCfg.get('USER','additional_input_files')
188 
189  if additionalInputFiles: additionalInputFiles += ',%s' % inputFile
190  else: additionalInputFiles = inputFile
191 
192  crabCfg.set('USER','additional_input_files',additionalInputFiles)
193 
194  return crabCfg
tools.addCrabInputFile
def addCrabInputFile
Definition: tools.py:183
def tools.copyFilesFromCastor (   castor_dir,
  output_dir,
  type = 'root' 
)

Definition at line 55 of file tools.py.

References listFilesInCastor().

55 
56 def copyFilesFromCastor(castor_dir,output_dir,type='root'):
57  from subprocess import call
58  files = listFilesInCastor(castor_dir,type,'')
59 
60  print "Copying files from %s to %s" % (castor_dir,output_dir)
61  for item in files:
62  cmd = ['rfcp',item,output_dir]
63  print "..." + item
64  retcode = call(cmd)
65  if retcode != 0: raise RuntimeError('Error in copying file %s to directory %s' % (item,output_dir))
66 
67  return 0
tools.listFilesInCastor
def listFilesInCastor
Definition: tools.py:30
tools.copyFilesFromCastor
def copyFilesFromCastor
Definition: tools.py:55
def tools.copyFilesLocal (   dir,
  output_dir,
  type = 'root' 
)

Definition at line 68 of file tools.py.

References listFilesLocal().

68 
69 def copyFilesLocal(dir,output_dir,type='root'):
70  if not dir: raise ValueError('Please specify valid dir')
71  if not output_dir: raise ValueError('Please specify valid output dir')
72 
73  from subprocess import call
74  files = listFilesLocal(dir,type)
75  cmd = ['cp']
76  cmd.extend(files)
77  cmd.append(output_dir)
78  print cmd
79  retcode = call(cmd)
80  return retcode
tools.listFilesLocal
def listFilesLocal
Definition: tools.py:42
tools.copyFilesLocal
def copyFilesLocal
Definition: tools.py:68
def tools.create_single_iov_db (   global_tag,
  run_number,
  output_db 
) 
Create an sqlite file with single-IOV tags for alignment payloads.

Arguments:
- `global_tag`: global tag from which to extract the payloads
- `run_number`: run for which the IOVs are selected
- `output_db`: name of the output sqlite file

Definition at line 7 of file tools.py.

References shell.connect(), join(), and run_checked().

7 
8 def create_single_iov_db(global_tag, run_number, output_db):
9  """Create an sqlite file with single-IOV tags for alignment payloads.
10 
11  Arguments:
12  - `global_tag`: global tag from which to extract the payloads
13  - `run_number`: run for which the IOVs are selected
14  - `output_db`: name of the output sqlite file
15  """
16 
17  con = shell.connect()
18  tags = con.global_tag_map(global_tag_name = global_tag,
19  record = ["TrackerAlignmentRcd",
20  "TrackerSurfaceDeformationRcd",
21  "TrackerAlignmentErrorExtendedRcd"])
22  con.close_session()
23 
24  tags = {item["record"]: {"name": item["tag_name"]}
25  for item in tags.as_dicts()}
26 
27  for record,tag in tags.iteritems():
28  iovs = con.tag(name = tag["name"]).iovs().as_dicts()
29  run_is_covered = False
30  for iov in reversed(iovs):
31  if iov["since"] <= run_number:
32  tag["since"] = str(iov["since"])
33  run_is_covered = True
34  break
35  if not run_is_covered:
36  msg = ("Run number {0:d} is not covered in '{1:s}' ({2:s}) from"
37  " '{3:s}'.".format(run_number, tag["name"], record,
38  global_tag))
39  print msg
40  print "Aborting..."
41  sys.exit(1)
42 
43  result = {}
44  if os.path.exists(output_db): os.remove(output_db)
45  for record,tag in tags.iteritems():
46  result[record] = {"connect": "sqlite_file:"+output_db,
47  "tag": "_".join([tag["name"], tag["since"]])}
48  cmd = ("conddb_import",
49  "-f", "frontier://PromptProd/cms_conditions",
50  "-c", result[record]["connect"],
51  "-i", tag["name"],
52  "-t", result[record]["tag"],
53  "-b", str(run_number),
54  "-e", str(run_number))
55  run_checked(cmd)
56  run_checked(["sqlite3", output_db, "update iov set since=1"])
57 
58  return result
59 
shell.connect
def connect
Definition: shell.py:10
tools.run_checked
def run_checked
Definition: tools.py:60
tools.create_single_iov_db
def create_single_iov_db
Definition: tools.py:7
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def tools.dqmWorkflowName (   datasetpath,
  type,
  rev = 1 
)

Definition at line 23 of file tools.py.

Referenced by DTDqm.DTDqm.initProcess().

23 
24 def dqmWorkflowName(datasetpath,type,rev=1):
25  workflowName = datasetpath
26  sections = workflowName.split('/')[1:]
27  workflowName = '/%s/%s-%s-rev%d/%s' % (sections[0],sections[1],type,rev,sections[2])
28 
29  return workflowName
tools.dqmWorkflowName
def dqmWorkflowName
Definition: tools.py:23
def tools.getDatasetStr (   datasetpath)

Definition at line 15 of file tools.py.

Referenced by DTDQMMerge.DTDQMMerge.initProcess(), DTCalibrationWorker.DTCalibrationWorker.runAnalysisResiduals(), DTCalibrationWorker.DTCalibrationWorker.runResidualCalib(), DTCalibrationWorker.DTCalibrationWorker.runTtrigProd(), DTCalibrationWorker.DTCalibrationWorker.runTtrigValid(), DTCalibrationWorker.DTCalibrationWorker.runValidationWorkflow(), DTCalibrationWorker.DTCalibrationWorker.runVDriftMeanTimerCalib(), and DTCalibrationWorker.DTCalibrationWorker.runVDriftSegmentCalib().

15 
16 def getDatasetStr(datasetpath):
17  datasetstr = datasetpath
18  datasetstr.strip()
19  if datasetstr[0] == '/': datasetstr = datasetstr[1:]
20  datasetstr = datasetstr.replace('/','_')
21 
22  return datasetstr
tools.getDatasetStr
def getDatasetStr
Definition: tools.py:15
def tools.haddInCastor (   castor_dir,
  result_file,
  type = 'root',
  prefix = 'rfio:',
  suffix = None 
)

Definition at line 81 of file tools.py.

References listFilesInCastor().

81 
82 def haddInCastor(castor_dir,result_file,type = 'root',prefix = 'rfio:',suffix = None):
83  if not castor_dir: raise ValueError('Please specify valid castor dir')
84  if not result_file: raise ValueError('Please specify valid output file name')
85 
86  #cmd = 'hadd %s `./listfilesCastor %s | grep %s`'%(result_file,castor_dir,type)
87  #print "Running",cmd
88  #os.system(cmd)
89  from subprocess import call
90  files = listFilesInCastor(castor_dir,type,prefix)
91  if suffix: files = [item + suffix for item in files]
92 
93  cmd = ['hadd',result_file]
94  cmd.extend(files)
95  #print cmd
96  retcode = call(cmd)
97  return retcode
tools.listFilesInCastor
def listFilesInCastor
Definition: tools.py:30
tools.haddInCastor
def haddInCastor
Definition: tools.py:81
def tools.haddLocal (   dir,
  result_file,
  type = 'root' 
)

Definition at line 98 of file tools.py.

References listFilesLocal().

98 
99 def haddLocal(dir,result_file,type = 'root'):
100  if not dir: raise ValueError('Please specify valid dir')
101  if not result_file: raise ValueError('Please specify valid output file name')
102 
103  from subprocess import call
104  files = listFilesLocal(dir,type)
105  cmd = ['hadd',result_file]
106  cmd.extend(files)
107  #print cmd
108  retcode = call(cmd)
109  return retcode
tools.listFilesLocal
def listFilesLocal
Definition: tools.py:42
tools.haddLocal
def haddLocal
Definition: tools.py:98
def tools.listFilesInCastor (   castor_dir,
  type = 'root',
  prefix = 'rfio:' 
)

Definition at line 30 of file tools.py.

Referenced by copyFilesFromCastor(), haddInCastor(), DTDqm.runDQM(), DTDQMMerge.runDQM(), and DTCalibrationWorker.DTCalibrationWorker.runDQMHarvesting().

30 
31 def listFilesInCastor(castor_dir,type = 'root',prefix = 'rfio:'):
32  if not castor_dir: raise ValueError('Please specify valid castor dir')
33 
34  from subprocess import Popen,PIPE
35  p1 = Popen(['nsls',castor_dir],stdout=PIPE)
36  #p2 = Popen(['grep',type],stdin=p1.stdout,stdout=PIPE)
37  #files = [prefix + castor_dir + "/" + item[:-1] for item in p2.stdout]
38  #p2.stdout.close()
39  files = [ "%s%s/%s" % (prefix,castor_dir,item.rstrip()) for item in p1.stdout if item.find(type) != -1 ]
40  p1.stdout.close()
41  return files
tools.listFilesInCastor
def listFilesInCastor
Definition: tools.py:30
def tools.listFilesLocal (   dir,
  type = 'root' 
)

Definition at line 42 of file tools.py.

Referenced by copyFilesLocal(), haddLocal(), DTCalibrationWorker.DTCalibrationWorker.runDQMHarvesting(), and DTCalibrationWorker.DTCalibrationWorker.runDQMMerge().

42 
43 def listFilesLocal(dir,type = 'root'):
44  if not dir: raise ValueError('Please specify valid dir')
45 
46  #from subprocess import Popen,PIPE
47  #p1 = Popen(['ls',dir],stdout=PIPE)
48  #p2 = Popen(['grep',type],stdin=p1.stdout,stdout=PIPE)
49  #files = [dir + "/" + item[:-1] for item in p2.stdout]
50  #p2.stdout.close()
51  files = os.listdir(dir)
52  files = [ "%s/%s" % (dir,item) for item in files if item.find(type) != -1 ]
53 
54  return files
tools.listFilesLocal
def listFilesLocal
Definition: tools.py:42
def tools.loadCmsProcess (   psetPath)

Definition at line 144 of file tools.py.

Referenced by DTValidSummary.DTValidSummary.initProcess(), DTDqm.DTDqm.initProcess(), DTDQMMerge.DTDQMMerge.initProcess(), DTAnalysisResiduals.DTAnalysisResiduals.initProcess(), DTTTrigProd.DTTTrigProd.initProcess(), DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initProcess(), DTDQMHarvesting.DTDQMHarvesting.initProcess(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initProcess(), DTNoiseCalibration.DTNoiseCalibration.initProcess(), DTResidualCalibration.DTResidualCalibration.initProcess(), DTDQMValidation.DTDQMValidation.initProcess(), and DTTTrigValid.DTTTrigValid.initProcess().

144 
145 def loadCmsProcess(psetPath):
146  module = __import__(psetPath)
147  process = sys.modules[psetPath].process
148 
149  import copy
150  #FIXME: clone process
151  #processNew = copy.deepcopy(process)
152  processNew = copy.copy(process)
153  return processNew
tools.loadCmsProcess
def loadCmsProcess
Definition: tools.py:144
def tools.loadCmsProcessFile (   psetName)

Definition at line 140 of file tools.py.

140 
141 def loadCmsProcessFile(psetName):
142  pset = imp.load_source("psetmodule",psetName)
143  return pset.process
tools.loadCmsProcessFile
def loadCmsProcessFile
Definition: tools.py:140
def tools.loadCrabCfg (   cfgName = None)

Definition at line 178 of file tools.py.

Referenced by DTNoiseCalibration.DTNoiseCalibration.initCrab(), DTTTrigProd.DTTTrigProd.initCrab(), DTDQMValidation.DTDQMValidation.initCrab(), DTResidualCalibration.DTResidualCalibration.initCrab(), DTAnalysisResiduals.DTAnalysisResiduals.initCrab(), DTTTrigValid.DTTTrigValid.initCrab(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initCrab(), and DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initCrab().

178 
179 def loadCrabCfg(cfgName=None):
180  config = ConfigParser.ConfigParser()
181  if cfgName: config.read(cfgName)
182  return config
tools.loadCrabCfg
def loadCrabCfg
Definition: tools.py:178
def tools.loadCrabDefault (   crabCfg,
  config 
)

Definition at line 195 of file tools.py.

Referenced by DTNoiseCalibration.DTNoiseCalibration.initCrab(), DTTTrigProd.DTTTrigProd.initCrab(), DTDQMValidation.DTDQMValidation.initCrab(), DTResidualCalibration.DTResidualCalibration.initCrab(), DTAnalysisResiduals.DTAnalysisResiduals.initCrab(), DTTTrigValid.DTTTrigValid.initCrab(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initCrab(), and DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initCrab().

195 
196 def loadCrabDefault(crabCfg,config):
197  # CRAB section
198  if not crabCfg.has_section('CRAB'): crabCfg.add_section('CRAB')
199  crabCfg.set('CRAB','jobtype','cmssw')
200 
201  if hasattr(config,'scheduler') and config.scheduler: crabCfg.set('CRAB','scheduler',config.scheduler)
202  else: crabCfg.set('CRAB','scheduler','CAF')
203 
204  if hasattr(config,'useserver') and config.useserver: crabCfg.set('CRAB','use_server',1)
205 
206  # CMSSW section
207  if not crabCfg.has_section('CMSSW'): crabCfg.add_section('CMSSW')
208  if hasattr(config,'datasetpath') and config.datasetpath: crabCfg.set('CMSSW','datasetpath',config.datasetpath)
209  else: crabCfg.set('CMSSW','datasetpath','/XXX/YYY/ZZZ')
210  crabCfg.set('CMSSW','pset','pset.py')
211 
212  # Splitting config
213  crabCfg.remove_option('CMSSW','total_number_of_events')
214  crabCfg.remove_option('CMSSW','events_per_job')
215  crabCfg.remove_option('CMSSW','number_of_jobs')
216  crabCfg.remove_option('CMSSW','total_number_of_lumis')
217  crabCfg.remove_option('CMSSW','lumis_per_job')
218  crabCfg.remove_option('CMSSW','lumi_mask')
219  crabCfg.remove_option('CMSSW','split_by_run')
220 
221  """
222  if hasattr(config,'totalnumberevents'): crabCfg.set('CMSSW','total_number_of_events',config.totalnumberevents)
223  if hasattr(config,'eventsperjob'): crabCfg.set('CMSSW','events_per_job',config.eventsperjob)
224  """
225  if hasattr(config,'splitByLumi') and config.splitByLumi:
226  crabCfg.set('CMSSW','total_number_of_lumis',config.totalnumberlumis)
227  crabCfg.set('CMSSW','lumis_per_job',config.lumisperjob)
228  if hasattr(config,'lumimask') and config.lumimask: crabCfg.set('CMSSW','lumi_mask',config.lumimask)
229  elif hasattr(config,'splitByEvent') and config.splitByEvent:
230  crabCfg.set('CMSSW','total_number_of_events',config.totalnumberevents)
231  crabCfg.set('CMSSW','events_per_job',config.eventsperjob)
232  else:
233  crabCfg.set('CMSSW','split_by_run',1)
234 
235  if hasattr(config,'splitByEvent') and config.splitByEvent:
236  crabCfg.remove_option('CMSSW','runselection')
237  else:
238  if hasattr(config,'runselection') and config.runselection:
239  crabCfg.set('CMSSW','runselection',config.runselection)
240 
241  # USER section
242  if not crabCfg.has_section('USER'): crabCfg.add_section('USER')
243 
244  # Stageout config
245  if hasattr(config,'stageOutCAF') and config.stageOutCAF:
246  crabCfg.set('USER','return_data',0)
247  crabCfg.set('USER','copy_data',1)
248  crabCfg.set('USER','storage_element','T2_CH_CAF')
249  crabCfg.set('USER','user_remote_dir',config.userdircaf)
250  crabCfg.set('USER','check_user_remote_dir',0)
251  elif hasattr(config,'stageOutLocal') and config.stageOutLocal:
252  crabCfg.set('USER','return_data',1)
253  crabCfg.set('USER','copy_data',0)
254  crabCfg.remove_option('USER','storage_element')
255  crabCfg.remove_option('USER','user_remote_dir')
256  crabCfg.remove_option('USER','check_user_remote_dir')
257 
258  if hasattr(config,'email') and config.email: crabCfg.set('USER','eMail',config.email)
259  crabCfg.set('USER','xml_report','crabReport.xml')
260 
261  if hasattr(config,'runOnGrid') and config.runOnGrid:
262  crabCfg.remove_section('CAF')
263  if hasattr(config,'ce_black_list'):
264  if not crabCfg.has_section('GRID'): crabCfg.add_section('GRID')
265  crabCfg.set('GRID','ce_black_list', config.ce_black_list)
266  if hasattr(config,'ce_white_list'):
267  if not crabCfg.has_section('GRID'): crabCfg.add_section('GRID')
268  crabCfg.set('GRID','ce_white_list', config.ce_white_list)
269  else:
270  if not crabCfg.has_section('CAF'): crabCfg.add_section('CAF')
271  crabCfg.set('CAF','queue',config.queueAtCAF)
272 
273  return crabCfg
tools.loadCrabDefault
def loadCrabDefault
Definition: tools.py:195
def tools.parseInput (   inputFields,
  requiredFields = () 
)

Definition at line 122 of file tools.py.

122 
123 def parseInput(inputFields,requiredFields = ()):
124 
125  class options: pass
126  for item in sys.argv:
127  option = item.split('=')[0]
128  if option in inputFields:
129  value = item.split('=')[1]
130  if value in ('true','True','yes','Yes'): value = True
131  elif value in ('false','False','no','No'): value = False
132 
133  setattr(options,option,value)
134 
135  for item in requiredFields:
136  if not hasattr(options,item):
137  raise RuntimeError('Need to set "%s"' % item)
138 
139  return options
tools.parseInput
def parseInput
Definition: tools.py:122
def tools.prependPaths (   process,
  seqname 
)

Definition at line 154 of file tools.py.

154 
155 def prependPaths(process,seqname):
156  for path in process.paths:
157  getattr(process,path)._seq = getattr(process,seqname)*getattr(process,path)._seq
tools.prependPaths
def prependPaths
Definition: tools.py:154
def tools.replaceTemplate (   template,
  opts 
)

Definition at line 5 of file tools.py.

References SiPixelLorentzAngle_cfi.read.

5 
6 def replaceTemplate(template,**opts):
7  result = open(template).read()
8  for item in opts:
9  old = '@@%s@@'%item
10  new = str(opts[item])
11  print "Replacing",old,"to",new
12  result = result.replace(old,new)
13 
14  return result
SiPixelLorentzAngle_cfi.read
tuple read
Definition: SiPixelLorentzAngle_cfi.py:7
tools.replaceTemplate
def replaceTemplate
Definition: tools.py:5
def tools.run_checked (   cmd) 
Run `cmd` and exit in case of failures.

Arguments:
- `cmd`: list containing the strings of the command

Definition at line 60 of file tools.py.

References join().

Referenced by create_single_iov_db().

60 
61 def run_checked(cmd):
62  """Run `cmd` and exit in case of failures.
63 
64  Arguments:
65  - `cmd`: list containing the strings of the command
66  """
67 
68  try:
69  with open(os.devnull, "w") as devnull:
70  subprocess.check_call(cmd, stdout = devnull)
71  except subprocess.CalledProcessError as e:
72  print "Problem in running the following command:"
73  print " ".join(e.cmd)
74  sys.exit(1)
tools.run_checked
def run_checked
Definition: tools.py:60
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def tools.setGridEnv (   cmssw_dir)

Definition at line 110 of file tools.py.

110 
111 def setGridEnv(cmssw_dir):
112  cwd = os.getcwd()
113  os.chdir(cmssw_dir)
114 
115  os.system('source /afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh')
116  os.system('cmsenv')
117  os.system('source /afs/cern.ch/cms/ccs/wm/scripts/Crab/crab.sh')
118 
119  os.chdir(cwd)
120 
121  return
tools.setGridEnv
def setGridEnv
Definition: tools.py:110
def tools.writeCfg (   process,
  dir,
  psetName 
)

Definition at line 158 of file tools.py.

References TablePrint.write.

158 
159 def writeCfg(process,dir,psetName):
160  if not os.path.exists(dir): os.makedirs(dir)
161  open(dir + '/' + psetName,'w').write(process.dumpPython())
tools.writeCfg
def writeCfg
Definition: tools.py:158
def tools.writeCfgPkl (   process,
  dir,
  psetName 
)

Definition at line 162 of file tools.py.

162 
163 def writeCfgPkl(process,dir,psetName):
164  if not os.path.exists(dir): os.makedirs(dir)
165 
166  pklFileName = psetName.split('.')[0] + '.pkl'
167  pklFile = open(dir + '/' + pklFileName,"wb")
168  myPickle = pickle.Pickler(pklFile)
169  myPickle.dump(process)
170  pklFile.close()
171 
172  outFile = open(dir + '/' + psetName,"w")
173  outFile.write("import FWCore.ParameterSet.Config as cms\n")
174  outFile.write("import pickle\n")
175  outFile.write("process = pickle.load(open('%s', 'rb'))\n" % pklFileName)
176  outFile.close()
177 
tools.writeCfgPkl
def writeCfgPkl
Definition: tools.py:162
Generated for CMSSW Reference Manual by doxygen 1.8.5