CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
mps_setup_new_align Namespace Reference

Functions

def add_campaign
 
def copy_default_templates
 
def customize_default_template
 
def get_first_match
 
def main
 

Variables

tuple required_version = (2,7)
 

Function Documentation

def mps_setup_new_align.add_campaign (   campaign_file,
  campaign,
  args 
)
Adds a line with campaign information from `args` to `campaign_file`.

Arguments:
- `campaign_file`: output file
- `campaign`: name of the campaign
- `args`: command line arguments for this campaign

Definition at line 151 of file mps_setup_new_align.py.

References submitPVValidationJobs.split().

Referenced by main().

152 def add_campaign(campaign_file, campaign, args):
153  """Adds a line with campaign information from `args` to `campaign_file`.
154 
155  Arguments:
156  - `campaign_file`: output file
157  - `campaign`: name of the campaign
158  - `args`: command line arguments for this campaign
159  """
160 
161  campaign_info = campaign.ljust(10)
162  campaign_info += os.environ["USER"].ljust(12)
163  campaign_info += datetime.date.today().isoformat().ljust(11)
164 
165  version = os.environ["CMSSW_VERSION"]
166  if args.checked_out[1]:
167  local_area = os.path.join(os.environ["CMSSW_BASE"], "src")
168  with open(os.devnull, 'w') as devnull:
169  # check which tags (-> e.g. release names) point at current commit
170  p = subprocess.Popen(["git", "tag", "--points-at", "HEAD"],
171  cwd = local_area, stdout=subprocess.PIPE,
172  stderr=devnull)
173  tags = p.communicate()[0].split()
174  # check for deleted, untracked, modified files respecting .gitignore:
175  p = subprocess.Popen(["git", "ls-files", "-d", "-o", "-m",
176  "--exclude-standard"],
177  cwd = local_area, stdout=subprocess.PIPE,
178  stderr=devnull)
179  files = p.communicate()[0].split()
180  # check for staged tracked files:
181  p = subprocess.Popen(["git", "diff", "--name-only", "--staged"],
182  cwd = local_area, stdout=subprocess.PIPE,
183  stderr=devnull)
184  files.extend(p.communicate()[0].split())
185  if version not in tags or len(files) != 0:
186  version += " (mod.)"
187 
188  campaign_info += version.ljust(34)
189  campaign_info += args.type.ljust(17)
190  campaign_info += args.description.strip() + "\n"
191 
192  fcntl.flock(campaign_file, fcntl.LOCK_EX)
193  campaign_file.write(campaign_info)
194  fcntl.flock(campaign_file, fcntl.LOCK_UN)
195 
196  return campaign_info
197 
def mps_setup_new_align.copy_default_templates (   args,
  next_campaign 
)
Copies the default configuration templates.

Arguments:
- `args`: container with the needed information
- `next_campaign`: destination for the copy operation

Definition at line 198 of file mps_setup_new_align.py.

References customize_default_template(), join(), and print().

Referenced by main().

199 def copy_default_templates(args, next_campaign):
200  """Copies the default configuration templates.
201 
202  Arguments:
203  - `args`: container with the needed information
204  - `next_campaign`: destination for the copy operation
205  """
206 
207  default_conf_dir = os.path.join(args.MPS_dir, "templates")
208  template_files = ("universalConfigTemplate.py", "alignment_config.ini")
209  for f in template_files:
210  shutil.copy(os.path.join(default_conf_dir, f), next_campaign)
211 
212  # customize alignment_config.ini
213  # - replace job name with campaign ID as initial value
214  # - replace global tag with the corresponding auto GT depending on data type
215  auto_gt = args.type.replace("MC", "phase1_2017_realistic")
216  auto_gt = auto_gt.replace("data", "run2_data")
217  customize_default_template(os.path.join(next_campaign, "alignment_config.ini"),
218  (r"(jobname\s*[=:])(.*)", r"\1 "+next_campaign),
219  (r"(globaltag\s*[=:])(.*)",
220  r"\1 auto:"+auto_gt))
221 
222  print(" - copied default configuration templates from", end=' ')
223  print("'"+default_conf_dir+"'")
224  print(" - please modify these template files according to your needs:", end=' ')
225  print(", ".join(template_files))
226 
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def mps_setup_new_align.customize_default_template (   file_name,
  regex_replace_pairs 
)
Replace all lines in `file_name` matching `regex_string` with
`replace_string`.
Lines starting with '#' or ';' are ignored.

Arguments:
- `file_name`: path to the file to be customized
- `regex_replace_pairs`: tuples containing a regex string and its
                         replacement

Definition at line 227 of file mps_setup_new_align.py.

Referenced by copy_default_templates(), and main().

228 def customize_default_template(file_name, *regex_replace_pairs):
229  """
230  Replace all lines in `file_name` matching `regex_string` with
231  `replace_string`.
232  Lines starting with '#' or ';' are ignored.
233 
234  Arguments:
235  - `file_name`: path to the file to be customized
236  - `regex_replace_pairs`: tuples containing a regex string and its
237  replacement
238  """
239 
240  comment = re.compile(r"^\s*[;#]")
241  replacements = []
242  for regex_string, replace_string in regex_replace_pairs:
243  replacements.append((re.compile(regex_string), replace_string))
244 
245  customized = ""
246  with open(file_name, "r") as f:
247  for line in f:
248  custom_line = line
249  if not re.match(comment, custom_line):
250  for regex,replace_string in replacements:
251  custom_line = re.sub(regex, replace_string, custom_line)
252  customized += custom_line
253  with open(file_name, "w") as f:
254  f.write(customized)
255 
def mps_setup_new_align.get_first_match (   regex,
  directory 
)
Checks if `directory` matches `regex` and returns the first match converted
to an integer. If it does not match -1 is returned.

Arguments:
- `regex`: Regular expression to be tested against
- `directory`: name of the directory under test

Definition at line 134 of file mps_setup_new_align.py.

Referenced by main().

135 def get_first_match(regex, directory):
136  """
137  Checks if `directory` matches `regex` and returns the first match converted
138  to an integer. If it does not match -1 is returned.
139 
140  Arguments:
141  - `regex`: Regular expression to be tested against
142  - `directory`: name of the directory under test
143  """
144 
145  result = regex.search(directory)
146  if result is None:
147  return -1
148  else:
149  return int(result.group(1))
150 
def mps_setup_new_align.main (   argv = None)
Main routine of the script.

Arguments:
- `argv`: arguments passed to the main routine

Definition at line 29 of file mps_setup_new_align.py.

References add_campaign(), helper.checked_out_MPS(), copy_default_templates(), customize_default_template(), get_first_match(), join(), print(), and str.

29 
30 def main(argv = None):
31  """Main routine of the script.
32 
33  Arguments:
34  - `argv`: arguments passed to the main routine
35  """
36 
37  if argv == None:
38  argv = sys.argv[1:]
39 
40  parser = argparse.ArgumentParser(
41  description="Setup a new alignment campaign in the MPproduction area.")
42  parser.add_argument("-d", "--description", dest="description", required=True,
43  help="comment to describe the purpose of the campaign")
44  parser.add_argument("-t", "--data-type", dest="type", required=True,
45  metavar="TYPE", choices=["MC", "data"],
46  help="type of the input data (choices: %(choices)s)")
47  parser.add_argument("-c", "--copy", dest="copy", metavar="CAMPAIGN",
48  help="input campaign (optional)")
49  args = parser.parse_args(argv)
50 
51 
52  if os.path.basename(os.path.normpath(os.getcwd())) != "MPproduction":
53  print(">>> Cannot create a campaign outside of the 'MPproduction' area.")
54  print(">>> Please change to the 'MPproduction' directory first.")
55  sys.exit(1)
56 
57  if len(args.description.strip()) == 0:
58  print(">>> Please provide a non-empty description of the campaign")
59  sys.exit(1)
60 
61  MPS_dir = os.path.join("src", "Alignment", "MillePedeAlignmentAlgorithm")
62  args.checked_out = checked_out_MPS()
63  if args.checked_out[0]:
64  MPS_dir = os.path.join(os.environ["CMSSW_BASE"], MPS_dir)
65  else:
66  MPS_dir = os.path.join(os.environ["CMSSW_RELEASE_BASE"], MPS_dir)
67  args.MPS_dir = MPS_dir
68 
69  mp_regex = re.compile(r"mp([0-9]+).*")
70  all_campaign_numbers = sorted(map(lambda x: get_first_match(mp_regex, x),
71  os.listdir(".")))
72  next_number = (0
73  if len(all_campaign_numbers) == 0
74  else sorted(all_campaign_numbers)[-1] + 1)
75 
76  while True:
77  try:
78  number_of_digits = len(str(next_number))
79  number_of_digits = 4 if number_of_digits <= 4 else number_of_digits
80  next_campaign = "mp{{0:0{0}d}}".format(number_of_digits)
81  next_campaign = next_campaign.format(next_number)
82  os.makedirs(next_campaign)
83  print(">>> Created new campaign:", next_campaign)
84 
85  campaign_list = "MP_ali_list.txt"
86  with open(campaign_list, "a") as f:
87  campaign_info = add_campaign(f, next_campaign, args)
88  backup_dir = ".MP_ali_list"
89  try:
90  os.makedirs(backup_dir)
91  except OSError as e:
92  if e.args == (17, 'File exists'):
93  pass
94  else:
95  raise
96  with open(os.path.join(backup_dir, campaign_list), "a") as f:
97  fcntl.flock(f, fcntl.LOCK_EX)
98  f.write(campaign_info)
99  fcntl.flock(f, fcntl.LOCK_UN)
100  print(" - updated campaign list '"+campaign_list+"'")
101 
102  if args.copy is None:
103  copy_default_templates(args, next_campaign)
104  else:
105  copied_files = []
106  for ext in ("py", "ini", "txt"):
107  for config_file in glob.glob(args.copy+"/*."+ext):
108  copied_files.append(os.path.basename(config_file))
109  shutil.copy(config_file, next_campaign)
110  if len(copied_files) == 0:
111  print(" - no configuration files for '"+args.copy+"'")
112  copy_default_templates(args, next_campaign)
113  else:
114  alignment_config_ini = os.path.join(next_campaign,
115  "alignment_config.ini")
116  regex_input = (r"^(jobname\s*[=:])(\s*)"+
117  os.path.basename(args.copy.strip("/"))+r"\s*$",
118  r"\1 "+next_campaign+r"\n")
119  if os.path.isfile(alignment_config_ini):
120  customize_default_template(alignment_config_ini,
121  regex_input)
122  print(" - copied configuration files from", end=' ')
123  print("'"+args.copy+"':", ", ".join(copied_files))
124 
125  except OSError as e:
126  if e.args == (17, 'File exists'):
127  next_number += 1 # someone created a campaign ~at the same time
128  continue
129  else:
130  raise
131  break
132 
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def checked_out_MPS
Definition: helper.py:5
#define str(s)

Variable Documentation

tuple mps_setup_new_align.required_version = (2,7)

Definition at line 20 of file mps_setup_new_align.py.