3 from __future__
import print_function
7 if sys.version_info[0]>2:
8 import _pickle
as cPickle
15 import configparser
as ConfigParser
16 import Alignment.MillePedeAlignmentAlgorithm.mpslib.tools
as mps_tools
17 import Alignment.MillePedeAlignmentAlgorithm.mpslib.Mpslibclass
as mpslib
18 import Alignment.MillePedeAlignmentAlgorithm.mpsvalidate.iniparser
as mpsv_iniparser
19 import Alignment.MillePedeAlignmentAlgorithm.mpsvalidate.trackerTree
as mpsv_trackerTree
20 from Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.helper
import checked_out_MPS
21 from functools
import reduce
27 """Main routine. Not called, if this module is loaded via `import`. 30 - `argv`: Command line arguments passed to the script. 37 setup_alignment.setup()
42 """Class encapsulating the alignment campaign setup procedure.""" 48 - `argv`: command line arguments 79 """Setup the alignment campaign.""" 87 if self._override_gt.strip() !=
"":
88 msg = (
"Overriding global tag with single-IOV tags extracted from " 98 """Create ConfigParser object from command line arguments.""" 100 helpEpilog =
"""Builds the config-templates from a universal 101 config-template for each dataset specified in .ini-file that is passed 102 to this script. Then calls mps_setup.py for all datasets.""" 103 parser = argparse.ArgumentParser(
104 description = (
"Setup the alignment as configured in the " 105 "alignment_config file."),
107 parser.add_argument(
"-v",
"--verbose", action=
"store_true",
108 help=
"display detailed output of mps_setup")
109 parser.add_argument(
"-w",
"--weight", action=
"store_true",
110 help=(
"creates additional merge job(s) with " 111 "(possibly new) weights from .ini-config"))
112 parser.add_argument(
"alignmentConfig",
113 help=(
"name of the .ini config file that specifies " 114 "the datasets to be used"))
117 self.
_config = ConfigParser.ConfigParser()
118 self._config.optionxform = str
120 self._config.read(self._args.alignmentConfig)
121 self._config.config_path = self._args.alignmentConfig
125 """Determine directory paths and create the ones that are needed.""" 127 mpsTemplates = os.path.join(
"src",
"Alignment",
128 "MillePedeAlignmentAlgorithm",
"templates")
130 mpsTemplates = os.path.join(os.environ[
"CMSSW_BASE"], mpsTemplates)
132 mpsTemplates = os.path.join(os.environ[
"CMSSW_RELEASE_BASE"], mpsTemplates)
133 self.
_mille_script = os.path.join(mpsTemplates,
"mps_runMille_template.sh")
134 self.
_pede_script = os.path.join(mpsTemplates,
"mps_runPede_rfcp_template.sh")
137 currentDir = os.getcwd()
138 match = re.search(re.compile(
'mpproduction\/mp(.+?)$', re.M|re.I),currentDir)
142 print(
"Current location does not seem to be a MillePede campaign directory:", end=
' ')
148 """Create and fill `general_options` dictionary.""" 158 """Fetch information about external datasets.""" 160 if self._config.has_option(
"general",
"externalDatasets"):
161 datasets =
map(
lambda x: x.strip(),
162 self._config.get(
"general",
163 "externalDatasets").
split(
","))
164 datasets = [x
for x
in datasets
if len(x.strip()) > 0]
165 for item
in datasets:
166 splitted = item.split(
"|")
167 dataset = splitted[0].
strip()
168 dataset = os.path.expandvars(dataset)
170 weight = splitted[1]
if len(splitted) > 1
else None 171 config = ConfigParser.ConfigParser()
172 config.optionxform = str
174 config.config_path = dataset
182 Create MPS mass storage directory where, e.g., mille binaries are 187 self.
_mss_dir = self._general_options.get(
"massStorageDir",
188 "/eos/cms/store/group/alca_millepede/")
192 cmd = [
"mkdir",
"-p", self.
_mss_dir]
195 if not self._general_options.get(
"testMode",
False):
197 with open(os.devnull,
"w")
as dump:
198 subprocess.check_call(cmd, stdout = dump, stderr = dump)
199 except subprocess.CalledProcessError:
200 print(
"Failed to create mass storage directory:", self.
_mss_dir)
205 """Extract different weight configurations from `self._config`.""" 207 weights_list = [[(name, weight)
for weight
in self.
_weight_dict[name]]
210 common_weights_list = [[(name, weight)
214 common_weights_dicts = []
215 for item
in itertools.product(*common_weights_list):
217 for name,weight
in item:
219 common_weights_dicts.append(d)
222 for weight_conf
in itertools.product(*weights_list):
223 number_of_configs = len(weight_configs)
224 for common_weight
in common_weights_dicts:
226 = tuple([(dataset[0],
227 reduce(
lambda x,y: mps_tools.replace_factors(x, y, common_weight[y]),
228 common_weight, dataset[1]))
229 for dataset
in weight_conf])
230 if replaced_config
not in weight_configs:
231 weight_configs.append(replaced_config)
234 if len(weight_configs) == number_of_configs:
235 weight_configs.append(weight_conf)
237 for weight_config
in weight_configs:
238 resolved_weight_config \
239 = [(dataset[0], mps_tools.compute_product_string(dataset[1]))
240 for dataset
in weight_config]
241 self._weight_configs.append(resolved_weight_config)
245 """Fetch 'pedesettings' from general section in `self._config`.""" 247 self._pede_settings \
249 for x
in self._config.get(
"general",
"pedesettings").
split(
",")]
250 if self._config.has_option(
"general",
"pedesettings")
else [
None])
254 """Create the mille jobs based on the [dataset:<name>] sections.""" 256 gt_regex = re.compile(
'setupGlobaltag\s*\=\s*[\"\'](.*?)[\"\']')
257 sg_regex = re.compile(
"setupRunStartGeometry\s*\=\s*.*$", re.M)
258 collection_regex = re.compile(
'setupCollection\s*\=\s*[\"\'](.*?)[\"\']')
259 czt_regex = re.compile(
'setupCosmicsZeroTesla\s*\=\s*.*$', re.M)
260 cdm_regex = re.compile(
'setupCosmicsDecoMode\s*\=\s*.*$', re.M)
261 pw_regex = re.compile(
'setupPrimaryWidth\s*\=\s*.*$', re.M)
262 json_regex = re.compile(
'setupJson\s*\=\s*.*$', re.M)
265 for name, dataset
in six.iteritems(self.
_datasets):
269 with open(dataset[
"configTemplate"],
"r") as f: 272 print(
"The config-template called", end=
' ')
273 print(dataset[
"configTemplate"],
"cannot be found.")
276 tmpFile = re.sub(gt_regex,
277 'setupGlobaltag = \"'+dataset[
"globaltag"]+
'\"',
279 tmpFile = re.sub(sg_regex,
280 "setupRunStartGeometry = "+
282 tmpFile = re.sub(collection_regex,
283 'setupCollection = \"'+dataset[
"collection"]+
'\"',
285 if dataset[
'cosmicsZeroTesla']:
286 tmpFile = re.sub(czt_regex,
287 'setupCosmicsZeroTesla = True',
289 if dataset[
'cosmicsDecoMode']:
290 tmpFile = re.sub(cdm_regex,
291 'setupCosmicsDecoMode = True',
293 if dataset[
'primaryWidth'] > 0.0:
294 tmpFile = re.sub(pw_regex,
295 'setupPrimaryWidth = '+
str(dataset[
"primaryWidth"]),
297 if dataset[
'json'] !=
'':
298 tmpFile = re.sub(json_regex,
299 'setupJson = \"'+dataset[
"json"]+
'\"',
302 thisCfgTemplate =
"tmp.py" 303 with open(thisCfgTemplate,
"w")
as f:
311 first_dataset =
False 313 self.
_cms_process = mps_tools.get_process_object(thisCfgTemplate)
316 with open(thisCfgTemplate,
"a")
as f: f.write(self.
_override_gt)
320 command = [
"mps_setup.py",
327 dataset[
"inputFileList"],
328 str(dataset[
"njobs"]),
333 if dataset[
"numberOfEvents"] > 0:
334 command.extend([
"--max-events",
str(dataset[
"numberOfEvents"])])
335 command = [x
for x
in command
if len(x.strip()) > 0]
338 print(
"Creating jobs for dataset:", name)
340 print(
"Baseconfig: ", dataset[
"configTemplate"])
341 print(
"Collection: ", dataset[
"collection"])
342 if dataset[
"collection"]
in (
"ALCARECOTkAlCosmicsCTF0T",
343 "ALCARECOTkAlCosmicsInCollisions"):
344 print(
"cosmicsDecoMode: ", dataset[
"cosmicsDecoMode"])
345 print(
"cosmicsZeroTesla: ", dataset[
"cosmicsZeroTesla"])
346 print(
"Globaltag: ", dataset[
"globaltag"])
347 print(
"Number of jobs: ", dataset[
"njobs"])
348 print(
"Inputfilelist: ", dataset[
"inputFileList"])
349 if dataset[
"json"] !=
"":
350 print(
"Jsonfile: ", dataset[
"json"])
351 if self._args.verbose:
352 print(
"Pass to mps_setup: ",
" ".
join(command))
362 """Create pede jobs from the given input.""" 364 for setting
in self._pede_settings:
369 "s" if len(self._pede_settings)*len(self.
_weight_configs) > 1
else ""))
372 print(
"Creating pede jobs using settings from '{0}'.".
format(setting))
377 thisCfgTemplate =
"tmp.py" 380 self.
_cms_process = mps_tools.get_process_object(thisCfgTemplate)
382 with open(thisCfgTemplate,
"a")
as f: f.write(self.
_override_gt)
384 for name,weight
in weight_conf:
392 lib = mpslib.jobdatabase()
396 jobm_path = os.path.join(
"jobData", lib.JOBDIR[-1])
399 command = [
"rm",
"-f", os.path.join(jobm_path,
"alignment_merge.py")]
405 "-w", thisCfgTemplate,
406 os.path.join(jobm_path,
"alignment_merge.py"),
410 if setting
is not None: command.extend([
"-a", setting])
417 os.path.abspath(os.path.join(jobm_path,
418 ".TrackerTree.root")))
422 with open(os.path.join(jobm_path,
".weights.pkl"),
"wb")
as f:
423 cPickle.dump(weight_conf, f, 2)
432 Create pede jobs in addition to already existing ones. Return GT 437 if not os.path.isdir(
"jobData"):
438 print(
"No jobData-folder found.", end=
' ')
439 print(
"Properly set up the alignment before using the -w option.")
441 if not os.path.exists(
"mps.db"):
442 print(
"No mps.db found.", end=
' ')
443 print(
"Properly set up the alignment before using the -w option.")
447 config_template = firstDataset[
"configTemplate"]
448 collection = firstDataset[
"collection"]
451 with open(config_template,
"r") as f: 454 print(
"The config-template '"+config_template+
"' cannot be found.")
457 tmpFile = re.sub(
'setupGlobaltag\s*\=\s*[\"\'](.*?)[\"\']',
460 tmpFile = re.sub(
'setupCollection\s*\=\s*[\"\'](.*?)[\"\']',
461 'setupCollection = \"'+collection+
'\"',
463 tmpFile = re.sub(re.compile(
"setupRunStartGeometry\s*\=\s*.*$", re.M),
475 Wrapper around subprocess calls which treats output depending on verbosity 479 - `command`: list of command items 480 - `verbose`: flag to turn on verbosity 483 call_method = subprocess.check_call
if verbose
else subprocess.check_output
485 call_method(command, stderr=subprocess.STDOUT)
486 except subprocess.CalledProcessError
as e:
487 print(
"" if verbose
else e.output)
488 print(
"Failed to execute command:",
" ".
join(command))
494 Create sqlite file with single-IOV tags and use it to override the 495 GT. If the GT is already customized by the user, the customization has 496 higher priority. Creates a snippet to be appended to the configuration 501 if not run_number > 0:
502 print(
"'FirstRunForStartGeometry' must be positive, but is", run_number)
505 input_db_name = os.path.abspath(
"alignment_input.db")
507 run_number, input_db_name)
510 for record,tag
in six.iteritems(tags):
514 "Alignment.MillePedeAlignmentAlgorithm.alignmentsetup." 515 "SetCondition as tagwriter\n")
516 self.
_override_gt += (
"\ntagwriter.setCondition(process,\n" 517 " connect = \""+tag[
"connect"]+
"\",\n" 518 " record = \""+record+
"\",\n" 519 " tag = \""+tag[
"tag"]+
"\")\n")
524 Check consistency of input alignment payloads and IOV definition. 525 Returns a dictionary with the information needed to override possibly 526 problematic input taken from the global tag. 529 print(
"Checking consistency of IOV definition...")
530 iovs = mps_tools.make_unique_runranges(self._cms_process.AlignmentProducer)
533 "TrackerAlignmentRcd":
None,
534 "TrackerSurfaceDeformationRcd":
None,
535 "TrackerAlignmentErrorExtendedRcd":
None,
538 for condition
in self._cms_process.GlobalTag.toGet.value():
539 if condition.record.value()
in inputs:
540 inputs[condition.record.value()] = {
541 "tag": condition.tag.value(),
543 if not condition.hasParameter(
"connect")
544 else condition.connect.value())
547 inputs_from_gt = [record
for record
in inputs
if inputs[record]
is None]
549 mps_tools.get_tags(self._cms_process.GlobalTag.globaltag.value(),
553 if iovs[0] == 1
and len(iovs) == 1:
554 print(
"Single IOV output detected in configuration and", end=
' ')
555 print(
"'FirstRunForStartGeometry' is not 1.")
556 print(
"Creating single IOV output from input conditions in run", end=
' ')
558 for inp
in inputs: inputs[inp][
"problematic"] =
True 560 print(
"Value of 'FirstRunForStartGeometry' has to match first", end=
' ')
561 print(
"defined output IOV:", end=
' ')
565 for inp
in six.itervalues(inputs):
566 inp[
"iovs"] = mps_tools.get_iovs(inp[
"connect"], inp[
"tag"])
569 problematic_gt_inputs = {}
570 input_indices = {key: len(value[
"iovs"]) -1
571 for key,value
in six.iteritems(inputs)}
572 for iov
in reversed(iovs):
574 if inputs[inp].pop(
"problematic",
False):
575 problematic_gt_inputs[inp] = inputs[inp]
576 if inp
in problematic_gt_inputs:
continue 577 if input_indices[inp] < 0:
578 print(
"First output IOV boundary at run", iov, end=
' ')
579 print(
"is before the first input IOV boundary at", end=
' ')
580 print(inputs[inp][
"iovs"][0],
"for '"+inp+
"'.")
581 print(
"Please check your run range selection.")
583 input_iov = inputs[inp][
"iovs"][input_indices[inp]]
585 if inp
in inputs_from_gt:
586 problematic_gt_inputs[inp] = inputs[inp]
587 print(
"Found problematic input taken from global tag.")
588 print(
"Input IOV boundary at run",input_iov, end=
' ')
589 print(
"for '"+inp+
"' is within output IOV starting", end=
' ')
591 print(
"Deriving an alignment with coarse IOV", end=
' ')
592 print(
"granularity starting from finer granularity", end=
' ')
593 print(
"leads to wrong results.")
594 print(
"A single IOV input using the IOV of", end=
' ')
596 print(
"is automatically created and used.")
598 print(
"Found input IOV boundary at run",input_iov, end=
' ')
599 print(
"for '"+inp+
"' which is within output IOV starting", end=
' ')
601 print(
"Deriving an alignment with coarse IOV granularity", end=
' ')
602 print(
"starting from finer granularity leads to wrong", end=
' ')
604 print(
"Please check your run range selection.")
606 elif iov == input_iov:
607 input_indices[inp] -= 1
610 input_indices = {key: len(value[
"iovs"]) -1
611 for key,value
in six.iteritems(inputs)
612 if (key !=
"TrackerAlignmentRcd")
613 and (inp
not in problematic_gt_inputs)}
614 for iov
in reversed(inputs[
"TrackerAlignmentRcd"][
"iovs"]):
615 for inp
in input_indices:
616 input_iov = inputs[inp][
"iovs"][input_indices[inp]]
618 print(
"Found input IOV boundary at run",input_iov, end=
' ')
619 print(
"for '"+inp+
"' which is within 'TrackerAlignmentRcd'", end=
' ')
620 print(
"IOV starting with run",
str(iov)+
".")
621 print(
"Deriving an alignment with inconsistent IOV boundaries", end=
' ')
622 print(
"leads to wrong results.")
623 print(
"Please check your input IOVs.")
625 elif iov == input_iov:
626 input_indices[inp] -= 1
628 print(
" -> IOV consistency check successful.")
631 return problematic_gt_inputs
635 """Method to create hidden 'TrackerTree.root'.""" 638 print(
"Trying to create the tracker tree before setting the global", end=
' ')
639 print(
"tag or the run to determine the geometry IOV.")
642 config = mpsv_iniparser.ConfigData()
643 config.jobDataPath =
"." 650 """Fetch general options from config file.""" 652 for var
in (
"classInf",
"pedeMem",
"jobname",
"FirstRunForStartGeometry"):
655 except ConfigParser.NoOptionError:
656 print(
"No", var,
"found in [general] section.", end=
' ')
657 print(
"Please check ini-file.")
663 """Fetch default general options from config file.""" 665 for var
in (
"globaltag",
"configTemplate",
"json",
"massStorageDir",
669 except ConfigParser.NoOptionError:
670 if var ==
"testMode":
continue 671 print(
"No '" + var +
"' given in [general] section.")
674 dataset[
"general"] = {}
675 for var
in (
"globaltag",
"configTemplate",
"json"):
677 dataset[
"general"][var] = dataset[
"config"].
get(
"general", var)
678 except (ConfigParser.NoSectionError,ConfigParser.NoOptionError):
684 Fetch 'datasetDir' variable from general section and add it to the 685 'os.environ' dictionary. 688 if self._config.has_option(
"general",
"datasetdir"):
689 dataset_directory = self._config.get(
"general",
"datasetdir")
691 os.environ[
"datasetdir"] = dataset_directory
694 print(
"No datasetdir given in [general] section.", end=
' ')
695 print(
"Be sure to give a full path in inputFileList.")
700 """Fetch internal and external dataset configurations.""" 702 all_configs = collections.OrderedDict()
703 all_configs[
"main"] = {
"config": self.
_config,
708 for config
in six.itervalues(all_configs):
709 global_weight =
"1" if config[
"weight"]
is None else config[
"weight"]
712 self._config.config_path]
716 global_weight = (global_weight,)
719 for section
in config[
"config"].sections():
720 cache_datasetdir = os.environ[
"datasetdir"]
721 if "general" in section:
722 if config[
"config"].has_option(
"general",
"datasetdir"):
723 os.environ[
"datasetdir"] = config[
"config"].
get(
"general",
"datasetdir")
724 elif section ==
"weights":
725 for option
in config[
"config"].
options(section):
726 common_weights[option] \
727 = [x.strip()
for x
in 728 config[
"config"].
get(section, option).
split(
",")]
729 elif section.startswith(
"dataset:"):
734 print(
"WARNING: Duplicate definition of dataset '{}'".
format(name))
735 print(
" -> Using defintion in '{}':\n".
format(config[
"config"].config_path))
737 for k,v
in config[
"config"].
items(section):
738 print(
" ", k,
"=", v)
743 if config[
"config"].has_option(section,
"weight"):
745 = [x.strip()
for x
in 746 config[
"config"].
get(section,
"weight").
split(
",")]
751 for global_w
in global_weight]
755 for var
in (
"inputFileList",
"collection"):
757 self.
_datasets[name][var] = config[
"config"].
get(section, var)
758 except ConfigParser.NoOptionError:
759 print(
"No", var,
"found in", section+
". Please check ini-file.")
764 for var
in (
"configTemplate",
"globaltag"):
766 self.
_datasets[name][var] = config[
"config"].
get(section, var)
767 except (ConfigParser.NoSectionError,ConfigParser.NoOptionError):
769 self.
_datasets[name][var] = config[
"general"][var]
773 = all_configs[
"main"][
"general"][var]
775 print(
"No",var,
"found in ["+section+
"]", end=
' ')
776 print(
"and no default in [general] section.")
780 self.
_datasets[name][
"cosmicsZeroTesla"] =
False 781 if config[
"config"].has_option(section,
"cosmicsZeroTesla"):
782 self.
_datasets[name][
"cosmicsZeroTesla"] \
783 = config[
"config"].getboolean(section,
"cosmicsZeroTesla")
785 self.
_datasets[name][
"cosmicsDecoMode"] =
False 786 if config[
"config"].has_option(section,
"cosmicsDecoMode"):
787 self.
_datasets[name][
"cosmicsDecoMode"] \
788 = config[
"config"].getboolean(section,
"cosmicsDecoMode")
790 self.
_datasets[name][
"primaryWidth"] = -1.0
791 if config[
"config"].has_option(section,
"primaryWidth"):
793 = config[
"config"].getfloat(section,
"primaryWidth")
795 self.
_datasets[name][
"numberOfEvents"] = -1
796 if config[
"config"].has_option(section,
"numberOfEvents"):
798 = config[
"config"].getint(section,
"numberOfEvents")
802 self.
_datasets[name][
"json"] = config[
"config"].
get(section,
"json")
803 except ConfigParser.NoOptionError:
805 self.
_datasets[name][
"json"] = config[
"general"][
"json"]
809 = all_configs[
"main"][
"general"][
"json"]
811 print(
"No json given in either [general] or", end=
' ')
812 print(
"["+section+
"] sections.")
813 print(
" -> Proceeding without json-file.")
817 for var
in (
"inputFileList",
"json",
"configTemplate"):
819 = os.path.expandvars(self.
_datasets[name][var])
825 with open(self.
_datasets[name][
"inputFileList"],
"r") as filelist: 826 for line
in filelist:
827 if "CastorPool" in line:
830 if not line.strip()==
"":
833 print(
"Inputfilelist", self.
_datasets[name][
"inputFileList"], end=
' ')
834 print(
"does not exist.")
837 print(
"Number of jobs is 0. There may be a problem with the inputfilelist:")
842 if config[
"config"].has_option(section,
"njobs"):
843 if config[
"config"].getint(section,
"njobs") <= self.
_datasets[name][
"njobs"]:
844 self.
_datasets[name][
"njobs"] = config[
"config"].getint(section,
"njobs")
846 print(
"'njobs' is bigger than the number of files for this", end=
' ')
848 print(
"Using default.")
850 print(
"No number of jobs specified. Using number of files in", end=
' ')
851 print(
"inputfilelist as the number of jobs.")
854 for weight_name, weight_values
in six.iteritems(common_weights):
855 for key, weight
in six.iteritems(weight_dict):
856 if any([weight_name
in w
for w
in weight]):
857 self.
_common_weights[weight_name+config[
"config"].config_path] = weight_values
860 weight_name+config[
"config"].config_path)
866 os.environ[
"datasetdir"] = cache_datasetdir
869 print(
"No dataset section defined in '{0}'".
format(
870 ", ".
join([self._args.aligmentConfig]+self._external_datasets.keys())))
871 print(
"At least one section '[dataset:<name>]' is required.")
878 if __name__ ==
"__main__":
881 except KeyboardInterrupt:
def _check_iov_definition(self)
std::vector< std::string_view > split(std::string_view, const char *)
def _create_mass_storage_directory(self)
def _create_input_db(self)
def _fetch_essentials(self)
bool any(const std::vector< T > &v, const T &what)
def _fetch_external_datasets(self)
def _fetch_pede_settings(self)
def _fetch_dataset_directory(self)
S & print(S &os, JobReport::InputFile const &f)
def _create_weight_configs(self)
def _fetch_defaults(self)
def _handle_process_call(self, command, verbose=False)
def _create_additional_pede_jobs(self)
def _create_mille_jobs(self)
static std::string join(char **cmd)
def _fill_general_options(self)
def _construct_paths(self)
def _create_tracker_tree(self)
def _fetch_datasets(self)
def _create_pede_jobs(self)