CMS 3D CMS Logo

mps_alisetup.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 from __future__ import print_function
4 import os
5 import re
6 import sys
7 if sys.version_info[0]>2:
8  import _pickle as cPickle
9 else:
10  import cPickle
11 import argparse
12 import itertools
13 import subprocess
14 import collections
15 import configparser as ConfigParser
16 import Alignment.MillePedeAlignmentAlgorithm.mpslib.tools as mps_tools
17 import Alignment.MillePedeAlignmentAlgorithm.mpslib.Mpslibclass as mpslib
18 import Alignment.MillePedeAlignmentAlgorithm.mpsvalidate.iniparser as mpsv_iniparser
19 import Alignment.MillePedeAlignmentAlgorithm.mpsvalidate.trackerTree as mpsv_trackerTree
20 from Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.helper import checked_out_MPS
21 from functools import reduce
22 
23 import six
24 
25 ################################################################################
26 def main(argv = None):
27  """Main routine. Not called, if this module is loaded via `import`.
28 
29  Arguments:
30  - `argv`: Command line arguments passed to the script.
31  """
32 
33  if argv == None:
34  argv = sys.argv[1:]
35 
36  setup_alignment = SetupAlignment(argv)
37  setup_alignment.setup()
38 
39 
40 ################################################################################
42  """Class encapsulating the alignment campaign setup procedure."""
43 
44  def __init__(self, argv):
45  """Constructor
46 
47  Arguments:
48  - `argv`: command line arguments
49  """
50 
51  self._argv = argv # raw command line arguments
52  self._args = None # parsed command line arguments
53  self._config = None # ConfigParser object
54  self._mss_dir = None # mass storage directory
55  self._datasets = collections.OrderedDict() # final dataset configs
56  self._first_run = None # first run for start geometry
57  self._cms_process = None # cms.Process extracted from CMSSW config
58  self._override_gt = None # snippet to append to config
59  self._pede_script = None # path to pede batch script template
60  self._weight_dict = collections.OrderedDict() # dictionary with dataset weights
61  self._mille_script = None # path to mille batch script template
62  self._mps_dir_name = None # MP campaign name (mp<ID>)
63  self._common_weights = {} # dictionary with dataset weights from [weights] section
64  self._weight_configs = [] # list with combinations of dataset weights
65  self._general_options = {} # general options extracted from ini file
66  self._external_datasets = collections.OrderedDict() # external dataset configs
67  self._first_pede_config = True # does a pede job exist already?
68 
69  self._create_config()
71  self._fetch_datasets()
72  self._construct_paths()
76 
77 
78  def setup(self):
79  """Setup the alignment campaign."""
80 
81  if self._args.weight:
83  else:
84  self._create_mille_jobs()
85  self._create_pede_jobs()
86 
87  if self._override_gt.strip() != "":
88  msg = ("Overriding global tag with single-IOV tags extracted from "
89  "'{}' for run number '{}'.".format(self._global_tag,
90  self._first_run))
91  print(msg)
92  print("-"*75)
93  print(self._override_gt)
94  print("="*75)
95 
96 
97  def _create_config(self):
98  """Create ConfigParser object from command line arguments."""
99 
100  helpEpilog ="""Builds the config-templates from a universal
101  config-template for each dataset specified in .ini-file that is passed
102  to this script. Then calls mps_setup.py for all datasets."""
103  parser = argparse.ArgumentParser(
104  description = ("Setup the alignment as configured in the "
105  "alignment_config file."),
106  epilog = helpEpilog)
107  parser.add_argument("-v", "--verbose", action="store_true",
108  help="display detailed output of mps_setup")
109  parser.add_argument("-w", "--weight", action="store_true",
110  help=("creates additional merge job(s) with "
111  "(possibly new) weights from .ini-config"))
112  parser.add_argument("alignmentConfig",
113  help=("name of the .ini config file that specifies "
114  "the datasets to be used"))
115 
116  self._args = parser.parse_args(self._argv)
117  self._config = ConfigParser.ConfigParser()
118  self._config.optionxform = str # default would give lowercase options
119  # -> not wanted
120  self._config.read(self._args.alignmentConfig)
121  self._config.config_path = self._args.alignmentConfig
122 
123 
124  def _construct_paths(self):
125  """Determine directory paths and create the ones that are needed."""
126 
127  mpsTemplates = os.path.join("src", "Alignment",
128  "MillePedeAlignmentAlgorithm", "templates")
129  if checked_out_MPS()[0]:
130  mpsTemplates = os.path.join(os.environ["CMSSW_BASE"], mpsTemplates)
131  else:
132  mpsTemplates = os.path.join(os.environ["CMSSW_RELEASE_BASE"], mpsTemplates)
133  self._mille_script = os.path.join(mpsTemplates, "mps_runMille_template.sh")
134  self._pede_script = os.path.join(mpsTemplates, "mps_runPede_rfcp_template.sh")
135 
136  # get working directory name
137  currentDir = os.getcwd()
138  match = re.search(re.compile('mpproduction\/mp(.+?)$', re.M|re.I),currentDir)
139  if match:
140  self._mps_dir_name = 'mp'+match.group(1)
141  else:
142  print("Current location does not seem to be a MillePede campaign directory:", end=' ')
143  print(currentDir)
144  sys.exit(1)
145 
146 
148  """Create and fill `general_options` dictionary."""
149 
150  print("="*75)
153  self._fetch_essentials()
154  self._fetch_defaults()
155 
156 
158  """Fetch information about external datasets."""
159 
160  if self._config.has_option("general", "externalDatasets"):
161  datasets = map(lambda x: x.strip(),
162  self._config.get("general",
163  "externalDatasets").split(","))
164  datasets = [x for x in datasets if len(x.strip()) > 0]
165  for item in datasets:
166  splitted = item.split("|")
167  dataset = splitted[0].strip()
168  dataset = os.path.expandvars(dataset)
169 
170  weight = splitted[1] if len(splitted) > 1 else None
171  config = ConfigParser.ConfigParser()
172  config.optionxform = str
173  config.read(dataset)
174  config.config_path = dataset
175  self._external_datasets[dataset] = {"config": config,
176  "weight": weight}
177 
178 
179 
181  """
182  Create MPS mass storage directory where, e.g., mille binaries are
183  stored.
184  """
185 
186  # set directory on eos
187  self._mss_dir = self._general_options.get("massStorageDir",
188  "/eos/cms/store/group/alca_millepede/")
189  self._mss_dir = os.path.join(self._mss_dir, "MPproductionFiles",
190  self._mps_dir_name)
191 
192  cmd = ["mkdir", "-p", self._mss_dir]
193 
194  # create directory
195  if not self._general_options.get("testMode", False):
196  try:
197  with open(os.devnull, "w") as dump:
198  subprocess.check_call(cmd, stdout = dump, stderr = dump)
199  except subprocess.CalledProcessError:
200  print("Failed to create mass storage directory:", self._mss_dir)
201  sys.exit(1)
202 
203 
205  """Extract different weight configurations from `self._config`."""
206 
207  weights_list = [[(name, weight) for weight in self._weight_dict[name]]
208  for name in self._weight_dict]
209 
210  common_weights_list = [[(name, weight)
211  for weight in self._common_weights[name]]
212  for name in self._common_weights]
213 
214  common_weights_dicts = []
215  for item in itertools.product(*common_weights_list):
216  d = {}
217  for name,weight in item:
218  d[name] = weight
219  common_weights_dicts.append(d)
220 
221  weight_configs = []
222  for weight_conf in itertools.product(*weights_list):
223  number_of_configs = len(weight_configs)
224  for common_weight in common_weights_dicts:
225  replaced_config \
226  = tuple([(dataset[0],
227  reduce(lambda x,y: mps_tools.replace_factors(x, y, common_weight[y]),
228  common_weight, dataset[1]))
229  for dataset in weight_conf])
230  if replaced_config not in weight_configs:
231  weight_configs.append(replaced_config)
232 
233  # default if config contains no common weights:
234  if len(weight_configs) == number_of_configs:
235  weight_configs.append(weight_conf)
236 
237  for weight_config in weight_configs:
238  resolved_weight_config \
239  = [(dataset[0], mps_tools.compute_product_string(dataset[1]))
240  for dataset in weight_config]
241  self._weight_configs.append(resolved_weight_config)
242 
243 
245  """Fetch 'pedesettings' from general section in `self._config`."""
246 
247  self._pede_settings \
248  = ([x.strip()
249  for x in self._config.get("general", "pedesettings").split(",")]
250  if self._config.has_option("general", "pedesettings") else [None])
251 
252 
254  """Create the mille jobs based on the [dataset:<name>] sections."""
255 
256  gt_regex = re.compile('setupGlobaltag\s*\=\s*[\"\'](.*?)[\"\']')
257  sg_regex = re.compile("setupRunStartGeometry\s*\=\s*.*$", re.M)
258  collection_regex = re.compile('setupCollection\s*\=\s*[\"\'](.*?)[\"\']')
259  czt_regex = re.compile('setupCosmicsZeroTesla\s*\=\s*.*$', re.M)
260  cdm_regex = re.compile('setupCosmicsDecoMode\s*\=\s*.*$', re.M)
261  pw_regex = re.compile('setupPrimaryWidth\s*\=\s*.*$', re.M)
262  json_regex = re.compile('setupJson\s*\=\s*.*$', re.M)
263 
264  first_dataset = True
265  for name, dataset in six.iteritems(self._datasets):
266  print("="*75)
267  # Build config from template/Fill in variables
268  try:
269  with open(dataset["configTemplate"],"r") as f:
270  tmpFile = f.read()
271  except IOError:
272  print("The config-template called", end=' ')
273  print(dataset["configTemplate"], "cannot be found.")
274  sys.exit(1)
275 
276  tmpFile = re.sub(gt_regex,
277  'setupGlobaltag = \"'+dataset["globaltag"]+'\"',
278  tmpFile)
279  tmpFile = re.sub(sg_regex,
280  "setupRunStartGeometry = "+
281  self._general_options["FirstRunForStartGeometry"], tmpFile)
282  tmpFile = re.sub(collection_regex,
283  'setupCollection = \"'+dataset["collection"]+'\"',
284  tmpFile)
285  if dataset['cosmicsZeroTesla']:
286  tmpFile = re.sub(czt_regex,
287  'setupCosmicsZeroTesla = True',
288  tmpFile)
289  if dataset['cosmicsDecoMode']:
290  tmpFile = re.sub(cdm_regex,
291  'setupCosmicsDecoMode = True',
292  tmpFile)
293  if dataset['primaryWidth'] > 0.0:
294  tmpFile = re.sub(pw_regex,
295  'setupPrimaryWidth = '+str(dataset["primaryWidth"]),
296  tmpFile)
297  if dataset['json'] != '':
298  tmpFile = re.sub(json_regex,
299  'setupJson = \"'+dataset["json"]+'\"',
300  tmpFile)
301 
302  thisCfgTemplate = "tmp.py"
303  with open(thisCfgTemplate, "w") as f:
304  f.write(tmpFile)
305 
306 
307  # Set mps_setup append option for datasets following the first one
308  append = "-a"
309  if first_dataset:
310  append = ""
311  first_dataset = False
312  self._config_template = tmpFile
313  self._cms_process = mps_tools.get_process_object(thisCfgTemplate)
314  self._create_input_db()
315 
316  with open(thisCfgTemplate, "a") as f: f.write(self._override_gt)
317 
318 
319  # create mps_setup command
320  command = ["mps_setup.py",
321  "-m",
322  append,
323  "-M", self._general_options["pedeMem"],
324  "-N", name,
325  self._mille_script,
326  thisCfgTemplate,
327  dataset["inputFileList"],
328  str(dataset["njobs"]),
329  self._general_options["classInf"],
330  self._general_options["jobname"],
331  self._pede_script,
332  "cmscafuser:"+self._mss_dir]
333  if dataset["numberOfEvents"] > 0:
334  command.extend(["--max-events", str(dataset["numberOfEvents"])])
335  command = [x for x in command if len(x.strip()) > 0]
336 
337  # Some output:
338  print("Creating jobs for dataset:", name)
339  print("-"*75)
340  print("Baseconfig: ", dataset["configTemplate"])
341  print("Collection: ", dataset["collection"])
342  if dataset["collection"] in ("ALCARECOTkAlCosmicsCTF0T",
343  "ALCARECOTkAlCosmicsInCollisions"):
344  print("cosmicsDecoMode: ", dataset["cosmicsDecoMode"])
345  print("cosmicsZeroTesla: ", dataset["cosmicsZeroTesla"])
346  print("Globaltag: ", dataset["globaltag"])
347  print("Number of jobs: ", dataset["njobs"])
348  print("Inputfilelist: ", dataset["inputFileList"])
349  if dataset["json"] != "":
350  print("Jsonfile: ", dataset["json"])
351  if self._args.verbose:
352  print("Pass to mps_setup: ", " ".join(command))
353 
354  # call the command and toggle verbose output
355  self._handle_process_call(command, self._args.verbose)
356 
357  # remove temporary file
358  self._handle_process_call(["rm", thisCfgTemplate])
359 
360 
361  def _create_pede_jobs(self):
362  """Create pede jobs from the given input."""
363 
364  for setting in self._pede_settings:
365  print()
366  print("="*75)
367  if setting is None:
368  print("Creating pede job{}.".format(
369  "s" if len(self._pede_settings)*len(self._weight_configs) > 1 else ""))
370  print("-"*75)
371  else:
372  print("Creating pede jobs using settings from '{0}'.".format(setting))
373  for weight_conf in self._weight_configs:
374  # blank weights
375  self._handle_process_call(["mps_weight.pl", "-c"])
376 
377  thisCfgTemplate = "tmp.py"
378  with open(thisCfgTemplate, "w") as f: f.write(self._config_template)
379  if self._override_gt is None:
380  self._cms_process = mps_tools.get_process_object(thisCfgTemplate)
381  self._create_input_db()
382  with open(thisCfgTemplate, "a") as f: f.write(self._override_gt)
383 
384  for name,weight in weight_conf:
385  self._handle_process_call(["mps_weight.pl", "-N", name, weight], True)
386 
387  if not self._first_pede_config:
388  # create new mergejob
389  self._handle_process_call(["mps_setupm.pl"], self._args.verbose)
390 
391  # read mps.db to find directory of new mergejob
392  lib = mpslib.jobdatabase()
393  lib.read_db()
394 
395  # short cut for jobm path
396  jobm_path = os.path.join("jobData", lib.JOBDIR[-1])
397 
398  # delete old merge-config
399  command = ["rm", "-f", os.path.join(jobm_path, "alignment_merge.py")]
400  self._handle_process_call(command, self._args.verbose)
401 
402  # create new merge-config
403  command = [
404  "mps_merge.py",
405  "-w", thisCfgTemplate,
406  os.path.join(jobm_path, "alignment_merge.py"),
407  jobm_path,
408  str(lib.nJobs),
409  ]
410  if setting is not None: command.extend(["-a", setting])
411  print("-"*75)
412  print(" ".join(command))
413  self._handle_process_call(command, self._args.verbose)
414  self._create_tracker_tree()
415  if self._first_pede_config:
416  os.symlink(self._tracker_tree_path,
417  os.path.abspath(os.path.join(jobm_path,
418  ".TrackerTree.root")))
419  self._first_pede_config = False
420 
421  # store weights configuration
422  with open(os.path.join(jobm_path, ".weights.pkl"), "wb") as f:
423  cPickle.dump(weight_conf, f, 2)
424  print("="*75)
425 
426  # remove temporary file
427  self._handle_process_call(["rm", thisCfgTemplate])
428 
429 
431  """
432  Create pede jobs in addition to already existing ones. Return GT
433  override snippet.
434  """
435 
436  # do some basic checks
437  if not os.path.isdir("jobData"):
438  print("No jobData-folder found.", end=' ')
439  print("Properly set up the alignment before using the -w option.")
440  sys.exit(1)
441  if not os.path.exists("mps.db"):
442  print("No mps.db found.", end=' ')
443  print("Properly set up the alignment before using the -w option.")
444  sys.exit(1)
445 
446  firstDataset = next(six.itervalues(self._datasets))
447  config_template = firstDataset["configTemplate"]
448  collection = firstDataset["collection"]
449 
450  try:
451  with open(config_template,"r") as f:
452  tmpFile = f.read()
453  except IOError:
454  print("The config-template '"+config_template+"' cannot be found.")
455  sys.exit(1)
456 
457  tmpFile = re.sub('setupGlobaltag\s*\=\s*[\"\'](.*?)[\"\']',
458  'setupGlobaltag = \"'+self._global_tag+'\"',
459  tmpFile)
460  tmpFile = re.sub('setupCollection\s*\=\s*[\"\'](.*?)[\"\']',
461  'setupCollection = \"'+collection+'\"',
462  tmpFile)
463  tmpFile = re.sub(re.compile("setupRunStartGeometry\s*\=\s*.*$", re.M),
464  "setupRunStartGeometry = "+self._first_run,
465  tmpFile)
466  self._config_template = tmpFile
467 
468  # first pede job exists already in this mode:
469  self._first_pede_config = False
470  self._create_pede_jobs()
471 
472 
473  def _handle_process_call(self, command, verbose = False):
474  """
475  Wrapper around subprocess calls which treats output depending on verbosity
476  level.
477 
478  Arguments:
479  - `command`: list of command items
480  - `verbose`: flag to turn on verbosity
481  """
482 
483  call_method = subprocess.check_call if verbose else subprocess.check_output
484  try:
485  call_method(command, stderr=subprocess.STDOUT)
486  except subprocess.CalledProcessError as e:
487  print("" if verbose else e.output)
488  print("Failed to execute command:", " ".join(command))
489  sys.exit(1)
490 
491 
492  def _create_input_db(self):
493  """
494  Create sqlite file with single-IOV tags and use it to override the
495  GT. If the GT is already customized by the user, the customization has
496  higher priority. Creates a snippet to be appended to the configuration
497  file.
498  """
499 
500  run_number = int(self._first_run)
501  if not run_number > 0:
502  print("'FirstRunForStartGeometry' must be positive, but is", run_number)
503  sys.exit(1)
504 
505  input_db_name = os.path.abspath("alignment_input.db")
506  tags = mps_tools.create_single_iov_db(self._check_iov_definition(),
507  run_number, input_db_name)
508 
509  self._override_gt = ""
510  for record,tag in six.iteritems(tags):
511  if self._override_gt == "":
512  self._override_gt \
513  += ("\nimport "
514  "Alignment.MillePedeAlignmentAlgorithm.alignmentsetup."
515  "SetCondition as tagwriter\n")
516  self._override_gt += ("\ntagwriter.setCondition(process,\n"
517  " connect = \""+tag["connect"]+"\",\n"
518  " record = \""+record+"\",\n"
519  " tag = \""+tag["tag"]+"\")\n")
520 
521 
523  """
524  Check consistency of input alignment payloads and IOV definition.
525  Returns a dictionary with the information needed to override possibly
526  problematic input taken from the global tag.
527  """
528 
529  print("Checking consistency of IOV definition...")
530  iovs = mps_tools.make_unique_runranges(self._cms_process.AlignmentProducer)
531 
532  inputs = {
533  "TrackerAlignmentRcd": None,
534  "TrackerSurfaceDeformationRcd": None,
535  "TrackerAlignmentErrorExtendedRcd": None,
536  }
537 
538  for condition in self._cms_process.GlobalTag.toGet.value():
539  if condition.record.value() in inputs:
540  inputs[condition.record.value()] = {
541  "tag": condition.tag.value(),
542  "connect": ("pro"
543  if not condition.hasParameter("connect")
544  else condition.connect.value())
545  }
546 
547  inputs_from_gt = [record for record in inputs if inputs[record] is None]
548  inputs.update(
549  mps_tools.get_tags(self._cms_process.GlobalTag.globaltag.value(),
550  inputs_from_gt))
551 
552  if int(self._first_run) != iovs[0]: # simple consistency check
553  if iovs[0] == 1 and len(iovs) == 1:
554  print("Single IOV output detected in configuration and", end=' ')
555  print("'FirstRunForStartGeometry' is not 1.")
556  print("Creating single IOV output from input conditions in run", end=' ')
557  print(self._first_run+".")
558  for inp in inputs: inputs[inp]["problematic"] = True
559  else:
560  print("Value of 'FirstRunForStartGeometry' has to match first", end=' ')
561  print("defined output IOV:", end=' ')
562  print(self._first_run, "!=", iovs[0])
563  sys.exit(1)
564 
565  for inp in six.itervalues(inputs):
566  inp["iovs"] = mps_tools.get_iovs(inp["connect"], inp["tag"])
567 
568  # check consistency of input with output
569  problematic_gt_inputs = {}
570  input_indices = {key: len(value["iovs"]) -1
571  for key,value in six.iteritems(inputs)}
572  for iov in reversed(iovs):
573  for inp in inputs:
574  if inputs[inp].pop("problematic", False):
575  problematic_gt_inputs[inp] = inputs[inp]
576  if inp in problematic_gt_inputs: continue
577  if input_indices[inp] < 0:
578  print("First output IOV boundary at run", iov, end=' ')
579  print("is before the first input IOV boundary at", end=' ')
580  print(inputs[inp]["iovs"][0], "for '"+inp+"'.")
581  print("Please check your run range selection.")
582  sys.exit(1)
583  input_iov = inputs[inp]["iovs"][input_indices[inp]]
584  if iov < input_iov:
585  if inp in inputs_from_gt:
586  problematic_gt_inputs[inp] = inputs[inp]
587  print("Found problematic input taken from global tag.")
588  print("Input IOV boundary at run",input_iov, end=' ')
589  print("for '"+inp+"' is within output IOV starting", end=' ')
590  print("with run", str(iov)+".")
591  print("Deriving an alignment with coarse IOV", end=' ')
592  print("granularity starting from finer granularity", end=' ')
593  print("leads to wrong results.")
594  print("A single IOV input using the IOV of", end=' ')
595  print("'FirstRunForStartGeometry' ("+self._first_run+")", end=' ')
596  print("is automatically created and used.")
597  continue
598  print("Found input IOV boundary at run",input_iov, end=' ')
599  print("for '"+inp+"' which is within output IOV starting", end=' ')
600  print("with run", str(iov)+".")
601  print("Deriving an alignment with coarse IOV granularity", end=' ')
602  print("starting from finer granularity leads to wrong", end=' ')
603  print("results.")
604  print("Please check your run range selection.")
605  sys.exit(1)
606  elif iov == input_iov:
607  input_indices[inp] -= 1
608 
609  # check consistency of 'TrackerAlignmentRcd' with other inputs
610  input_indices = {key: len(value["iovs"]) -1
611  for key,value in six.iteritems(inputs)
612  if (key != "TrackerAlignmentRcd")
613  and (inp not in problematic_gt_inputs)}
614  for iov in reversed(inputs["TrackerAlignmentRcd"]["iovs"]):
615  for inp in input_indices:
616  input_iov = inputs[inp]["iovs"][input_indices[inp]]
617  if iov < input_iov:
618  print("Found input IOV boundary at run",input_iov, end=' ')
619  print("for '"+inp+"' which is within 'TrackerAlignmentRcd'", end=' ')
620  print("IOV starting with run", str(iov)+".")
621  print("Deriving an alignment with inconsistent IOV boundaries", end=' ')
622  print("leads to wrong results.")
623  print("Please check your input IOVs.")
624  sys.exit(1)
625  elif iov == input_iov:
626  input_indices[inp] -= 1
627 
628  print(" -> IOV consistency check successful.")
629  print("="*75)
630 
631  return problematic_gt_inputs
632 
633 
635  """Method to create hidden 'TrackerTree.root'."""
636 
637  if self._global_tag is None or self._first_run is None:
638  print("Trying to create the tracker tree before setting the global", end=' ')
639  print("tag or the run to determine the geometry IOV.")
640  sys.exit(1)
641 
642  config = mpsv_iniparser.ConfigData()
643  config.jobDataPath = "." # current directory
644  config.globalTag = self._global_tag
645  config.firstRun = self._first_run
646  self._tracker_tree_path = mpsv_trackerTree.check(config)
647 
648 
649  def _fetch_essentials(self):
650  """Fetch general options from config file."""
651 
652  for var in ("classInf","pedeMem","jobname", "FirstRunForStartGeometry"):
653  try:
654  self._general_options[var] = self._config.get('general',var)
655  except ConfigParser.NoOptionError:
656  print("No", var, "found in [general] section.", end=' ')
657  print("Please check ini-file.")
658  sys.exit(1)
659  self._first_run = self._general_options["FirstRunForStartGeometry"]
660 
661 
662  def _fetch_defaults(self):
663  """Fetch default general options from config file."""
664 
665  for var in ("globaltag", "configTemplate", "json", "massStorageDir",
666  "testMode"):
667  try:
668  self._general_options[var] = self._config.get("general", var)
669  except ConfigParser.NoOptionError:
670  if var == "testMode": continue
671  print("No '" + var + "' given in [general] section.")
672 
673  for dataset in six.itervalues(self._external_datasets):
674  dataset["general"] = {}
675  for var in ("globaltag", "configTemplate", "json"):
676  try:
677  dataset["general"][var] = dataset["config"].get("general", var)
678  except (ConfigParser.NoSectionError,ConfigParser.NoOptionError):
679  pass
680 
681 
683  """
684  Fetch 'datasetDir' variable from general section and add it to the
685  'os.environ' dictionary.
686  """
687 
688  if self._config.has_option("general", "datasetdir"):
689  dataset_directory = self._config.get("general", "datasetdir")
690  # add it to environment for later variable expansion:
691  os.environ["datasetdir"] = dataset_directory
692  self._general_options["datasetdir"] = dataset_directory
693  else:
694  print("No datasetdir given in [general] section.", end=' ')
695  print("Be sure to give a full path in inputFileList.")
696  self._general_options["datasetdir"] = ""
697 
698 
699  def _fetch_datasets(self):
700  """Fetch internal and external dataset configurations."""
701 
702  all_configs = collections.OrderedDict()
703  all_configs["main"] = {"config": self._config,
704  "general": self._general_options,
705  "weight": None}
706  all_configs.update(self._external_datasets)
707 
708  for config in six.itervalues(all_configs):
709  global_weight = "1" if config["weight"] is None else config["weight"]
710  if global_weight+self._config.config_path in self._common_weights:
711  global_weight = self._common_weights[global_weight+
712  self._config.config_path]
713  elif global_weight in self._common_weights:
714  global_weight = self._common_weights[global_weight]
715  else:
716  global_weight = (global_weight,)
717  common_weights = {}
718  weight_dict = {}
719  for section in config["config"].sections():
720  cache_datasetdir = os.environ["datasetdir"]
721  if "general" in section:
722  if config["config"].has_option("general", "datasetdir"):
723  os.environ["datasetdir"] = config["config"].get("general", "datasetdir")
724  elif section == "weights":
725  for option in config["config"].options(section):
726  common_weights[option] \
727  = [x.strip() for x in
728  config["config"].get(section, option).split(",")]
729  elif section.startswith("dataset:"):
730  print("-"*75)
731  # set name from section-name
732  name = section[8:]
733  if name in self._datasets:
734  print("WARNING: Duplicate definition of dataset '{}'".format(name))
735  print(" -> Using defintion in '{}':\n".format(config["config"].config_path))
736  print(" [{}]".format(section))
737  for k,v in config["config"].items(section):
738  print(" ", k, "=", v)
739  print()
740  self._datasets[name] = {}
741 
742  # extract weight for the dataset
743  if config["config"].has_option(section, "weight"):
744  self._weight_dict[name] \
745  = [x.strip() for x in
746  config["config"].get(section, "weight").split(",")]
747  else:
748  self._weight_dict[name] = ["1.0"]
749  self._weight_dict[name] = [global_w+"*"+w
750  for w in self._weight_dict[name]
751  for global_w in global_weight]
752  weight_dict[name] = self._weight_dict[name]
753 
754  # extract essential variables
755  for var in ("inputFileList", "collection"):
756  try:
757  self._datasets[name][var] = config["config"].get(section, var)
758  except ConfigParser.NoOptionError:
759  print("No", var, "found in", section+". Please check ini-file.")
760  sys.exit(1)
761 
762  # get globaltag and configTemplate. If none in section, try to get
763  # default from [general] section.
764  for var in ("configTemplate", "globaltag"):
765  try:
766  self._datasets[name][var] = config["config"].get(section, var)
767  except (ConfigParser.NoSectionError,ConfigParser.NoOptionError):
768  try:
769  self._datasets[name][var] = config["general"][var]
770  except KeyError:
771  try:
772  self._datasets[name][var] \
773  = all_configs["main"]["general"][var]
774  except KeyError:
775  print("No",var,"found in ["+section+"]", end=' ')
776  print("and no default in [general] section.")
777  sys.exit(1)
778 
779  # extract non-essential options
780  self._datasets[name]["cosmicsZeroTesla"] = False
781  if config["config"].has_option(section,"cosmicsZeroTesla"):
782  self._datasets[name]["cosmicsZeroTesla"] \
783  = config["config"].getboolean(section,"cosmicsZeroTesla")
784 
785  self._datasets[name]["cosmicsDecoMode"] = False
786  if config["config"].has_option(section,"cosmicsDecoMode"):
787  self._datasets[name]["cosmicsDecoMode"] \
788  = config["config"].getboolean(section,"cosmicsDecoMode")
789 
790  self._datasets[name]["primaryWidth"] = -1.0
791  if config["config"].has_option(section,"primaryWidth"):
792  self._datasets[name]["primaryWidth"] \
793  = config["config"].getfloat(section,"primaryWidth")
794 
795  self._datasets[name]["numberOfEvents"] = -1
796  if config["config"].has_option(section, "numberOfEvents"):
797  self._datasets[name]["numberOfEvents"] \
798  = config["config"].getint(section, "numberOfEvents")
799 
800  self._datasets[name]["json"] = ""
801  try:
802  self._datasets[name]["json"] = config["config"].get(section,"json")
803  except ConfigParser.NoOptionError:
804  try:
805  self._datasets[name]["json"] = config["general"]["json"]
806  except KeyError:
807  try:
808  self._datasets[name]["json"] \
809  = all_configs["main"]["general"]["json"]
810  except KeyError:
811  print("No json given in either [general] or", end=' ')
812  print("["+section+"] sections.")
813  print(" -> Proceeding without json-file.")
814 
815 
816  #replace ${datasetdir} and other variables, e.g. $CMSSW_BASE
817  for var in ("inputFileList", "json", "configTemplate"):
818  self._datasets[name][var] \
819  = os.path.expandvars(self._datasets[name][var])
820 
821 
822  # Get number of jobs from lines in inputfilelist
823  self._datasets[name]["njobs"] = 0
824  try:
825  with open(self._datasets[name]["inputFileList"], "r") as filelist:
826  for line in filelist:
827  if "CastorPool" in line:
828  continue
829  # ignore empty lines
830  if not line.strip()=="":
831  self._datasets[name]["njobs"] += 1
832  except IOError:
833  print("Inputfilelist", self._datasets[name]["inputFileList"], end=' ')
834  print("does not exist.")
835  sys.exit(1)
836  if self._datasets[name]["njobs"] == 0:
837  print("Number of jobs is 0. There may be a problem with the inputfilelist:")
838  print(self._datasets[name]["inputFileList"])
839  sys.exit(1)
840 
841  # Check if njobs gets overwritten in .ini-file
842  if config["config"].has_option(section, "njobs"):
843  if config["config"].getint(section, "njobs") <= self._datasets[name]["njobs"]:
844  self._datasets[name]["njobs"] = config["config"].getint(section, "njobs")
845  else:
846  print("'njobs' is bigger than the number of files for this", end=' ')
847  print("dataset:", self._datasets[name]["njobs"])
848  print("Using default.")
849  else:
850  print("No number of jobs specified. Using number of files in", end=' ')
851  print("inputfilelist as the number of jobs.")
852 
853  # check if local weights override global weights and resolve name clashes
854  for weight_name, weight_values in six.iteritems(common_weights):
855  for key, weight in six.iteritems(weight_dict):
856  if any([weight_name in w for w in weight]):
857  self._common_weights[weight_name+config["config"].config_path] = weight_values
858  self._weight_dict[key] = [mps_tools.replace_factors(w,
859  weight_name,
860  weight_name+config["config"].config_path)
861  for w in weight]
862  else:
863  self._common_weights[weight_name] = weight_values
864  self._weight_dict[key] = weight
865 
866  os.environ["datasetdir"] = cache_datasetdir
867 
868  if len(self._datasets) == 0:
869  print("No dataset section defined in '{0}'".format(
870  ", ".join([self._args.aligmentConfig]+self._external_datasets.keys())))
871  print("At least one section '[dataset:<name>]' is required.")
872  sys.exit(1)
873 
874  self._global_tag = self._datasets[name]["globaltag"]
875 
876 
877 ################################################################################
878 if __name__ == "__main__":
879  try:
880  main()
881  except KeyboardInterrupt:
882  pass
std::vector< std::string_view > split(std::string_view, const char *)
def _create_mass_storage_directory(self)
def checked_out_MPS()
Definition: helper.py:5
bool any(const std::vector< T > &v, const T &what)
Definition: ECalSD.cc:40
def __init__(self, argv)
Definition: mps_alisetup.py:44
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
def _handle_process_call(self, command, verbose=False)
def _create_additional_pede_jobs(self)
static std::string join(char **cmd)
Definition: RemoteFile.cc:17
Definition: main.py:1
def main(argv=None)
Definition: mps_alisetup.py:26
#define str(s)