CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
Functions | Variables
mps_setup_new_align Namespace Reference

Functions

def add_campaign (campaign_file, campaign, args)
 
def copy_default_templates (args, next_campaign)
 
def customize_default_template (file_name, *regex_replace_pairs)
 
def get_first_match (regex, directory)
 
def main (argv=None)
 

Variables

tuple required_version = (2,7)
 

Function Documentation

◆ add_campaign()

def mps_setup_new_align.add_campaign (   campaign_file,
  campaign,
  args 
)
Adds a line with campaign information from `args` to `campaign_file`.

Arguments:
- `campaign_file`: output file
- `campaign`: name of the campaign
- `args`: command line arguments for this campaign

Definition at line 151 of file mps_setup_new_align.py.

151 def add_campaign(campaign_file, campaign, args):
152  """Adds a line with campaign information from `args` to `campaign_file`.
153 
154  Arguments:
155  - `campaign_file`: output file
156  - `campaign`: name of the campaign
157  - `args`: command line arguments for this campaign
158  """
159 
160  campaign_info = campaign.ljust(10)
161  campaign_info += os.environ["USER"].ljust(12)
162  campaign_info += datetime.date.today().isoformat().ljust(11)
163 
164  version = os.environ["CMSSW_VERSION"]
165  if args.checked_out[1]:
166  local_area = os.path.join(os.environ["CMSSW_BASE"], "src")
167  with open(os.devnull, 'w') as devnull:
168  # check which tags (-> e.g. release names) point at current commit
169  p = subprocess.Popen(["git", "tag", "--points-at", "HEAD"],
170  cwd = local_area, stdout=subprocess.PIPE,
171  stderr=devnull)
172  tags = p.communicate()[0].split()
173  # check for deleted, untracked, modified files respecting .gitignore:
174  p = subprocess.Popen(["git", "ls-files", "-d", "-o", "-m",
175  "--exclude-standard"],
176  cwd = local_area, stdout=subprocess.PIPE,
177  stderr=devnull)
178  files = p.communicate()[0].split()
179  # check for staged tracked files:
180  p = subprocess.Popen(["git", "diff", "--name-only", "--staged"],
181  cwd = local_area, stdout=subprocess.PIPE,
182  stderr=devnull)
183  files.extend(p.communicate()[0].split())
184  if version not in tags or len(files) != 0:
185  version += " (mod.)"
186 
187  campaign_info += version.ljust(34)
188  campaign_info += args.type.ljust(17)
189  campaign_info += args.description.strip() + "\n"
190 
191  fcntl.flock(campaign_file, fcntl.LOCK_EX)
192  campaign_file.write(campaign_info)
193  fcntl.flock(campaign_file, fcntl.LOCK_UN)
194 
195  return campaign_info
196 
197 

References submitPVValidationJobs.split().

Referenced by main().

◆ copy_default_templates()

def mps_setup_new_align.copy_default_templates (   args,
  next_campaign 
)
Copies the default configuration templates.

Arguments:
- `args`: container with the needed information
- `next_campaign`: destination for the copy operation

Definition at line 198 of file mps_setup_new_align.py.

198 def copy_default_templates(args, next_campaign):
199  """Copies the default configuration templates.
200 
201  Arguments:
202  - `args`: container with the needed information
203  - `next_campaign`: destination for the copy operation
204  """
205 
206  default_conf_dir = os.path.join(args.MPS_dir, "templates")
207  template_files = ("universalConfigTemplate.py", "alignment_config.ini")
208  for f in template_files:
209  shutil.copy(os.path.join(default_conf_dir, f), next_campaign)
210 
211  # customize alignment_config.ini
212  # - replace job name with campaign ID as initial value
213  # - replace global tag with the corresponding auto GT depending on data type
214  auto_gt = args.type.replace("MC", "phase1_2017_realistic")
215  auto_gt = auto_gt.replace("data", "run2_data")
216  customize_default_template(os.path.join(next_campaign, "alignment_config.ini"),
217  (r"(jobname\s*[=:])(.*)", r"\1 "+next_campaign),
218  (r"(globaltag\s*[=:])(.*)",
219  r"\1 auto:"+auto_gt))
220 
221  print(" - copied default configuration templates from", end=' ')
222  print("'"+default_conf_dir+"'")
223  print(" - please modify these template files according to your needs:", end=' ')
224  print(", ".join(template_files))
225 
226 

References customize_default_template(), join(), and print().

Referenced by main().

◆ customize_default_template()

def mps_setup_new_align.customize_default_template (   file_name,
regex_replace_pairs 
)
Replace all lines in `file_name` matching `regex_string` with
`replace_string`.
Lines starting with '#' or ';' are ignored.

Arguments:
- `file_name`: path to the file to be customized
- `regex_replace_pairs`: tuples containing a regex string and its
                         replacement

Definition at line 227 of file mps_setup_new_align.py.

227 def customize_default_template(file_name, *regex_replace_pairs):
228  """
229  Replace all lines in `file_name` matching `regex_string` with
230  `replace_string`.
231  Lines starting with '#' or ';' are ignored.
232 
233  Arguments:
234  - `file_name`: path to the file to be customized
235  - `regex_replace_pairs`: tuples containing a regex string and its
236  replacement
237  """
238 
239  comment = re.compile(r"^\s*[;#]")
240  replacements = []
241  for regex_string, replace_string in regex_replace_pairs:
242  replacements.append((re.compile(regex_string), replace_string))
243 
244  customized = ""
245  with open(file_name, "r") as f:
246  for line in f:
247  custom_line = line
248  if not re.match(comment, custom_line):
249  for regex,replace_string in replacements:
250  custom_line = re.sub(regex, replace_string, custom_line)
251  customized += custom_line
252  with open(file_name, "w") as f:
253  f.write(customized)
254 
255 

Referenced by copy_default_templates(), and main().

◆ get_first_match()

def mps_setup_new_align.get_first_match (   regex,
  directory 
)
Checks if `directory` matches `regex` and returns the first match converted
to an integer. If it does not match -1 is returned.

Arguments:
- `regex`: Regular expression to be tested against
- `directory`: name of the directory under test

Definition at line 134 of file mps_setup_new_align.py.

134 def get_first_match(regex, directory):
135  """
136  Checks if `directory` matches `regex` and returns the first match converted
137  to an integer. If it does not match -1 is returned.
138 
139  Arguments:
140  - `regex`: Regular expression to be tested against
141  - `directory`: name of the directory under test
142  """
143 
144  result = regex.search(directory)
145  if result is None:
146  return -1
147  else:
148  return int(result.group(1))
149 
150 

References createfilelist.int.

Referenced by main().

◆ main()

def mps_setup_new_align.main (   argv = None)
Main routine of the script.

Arguments:
- `argv`: arguments passed to the main routine

Definition at line 29 of file mps_setup_new_align.py.

29 def main(argv = None):
30  """Main routine of the script.
31 
32  Arguments:
33  - `argv`: arguments passed to the main routine
34  """
35 
36  if argv == None:
37  argv = sys.argv[1:]
38 
39  parser = argparse.ArgumentParser(
40  description="Setup a new alignment campaign in the MPproduction area.")
41  parser.add_argument("-d", "--description", dest="description", required=True,
42  help="comment to describe the purpose of the campaign")
43  parser.add_argument("-t", "--data-type", dest="type", required=True,
44  metavar="TYPE", choices=["MC", "data"],
45  help="type of the input data (choices: %(choices)s)")
46  parser.add_argument("-c", "--copy", dest="copy", metavar="CAMPAIGN",
47  help="input campaign (optional)")
48  args = parser.parse_args(argv)
49 
50 
51  if os.path.basename(os.path.normpath(os.getcwd())) != "MPproduction":
52  print(">>> Cannot create a campaign outside of the 'MPproduction' area.")
53  print(">>> Please change to the 'MPproduction' directory first.")
54  sys.exit(1)
55 
56  if len(args.description.strip()) == 0:
57  print(">>> Please provide a non-empty description of the campaign")
58  sys.exit(1)
59 
60  MPS_dir = os.path.join("src", "Alignment", "MillePedeAlignmentAlgorithm")
61  args.checked_out = checked_out_MPS()
62  if args.checked_out[0]:
63  MPS_dir = os.path.join(os.environ["CMSSW_BASE"], MPS_dir)
64  else:
65  MPS_dir = os.path.join(os.environ["CMSSW_RELEASE_BASE"], MPS_dir)
66  args.MPS_dir = MPS_dir
67 
68  mp_regex = re.compile(r"mp([0-9]+).*")
69  all_campaign_numbers = sorted(map(lambda x: get_first_match(mp_regex, x),
70  os.listdir(".")))
71  next_number = (0
72  if len(all_campaign_numbers) == 0
73  else sorted(all_campaign_numbers)[-1] + 1)
74 
75  while True:
76  try:
77  number_of_digits = len(str(next_number))
78  number_of_digits = 4 if number_of_digits <= 4 else number_of_digits
79  next_campaign = "mp{{0:0{0}d}}".format(number_of_digits)
80  next_campaign = next_campaign.format(next_number)
81  os.makedirs(next_campaign)
82  print(">>> Created new campaign:", next_campaign)
83 
84  campaign_list = "MP_ali_list.txt"
85  with open(campaign_list, "a") as f:
86  campaign_info = add_campaign(f, next_campaign, args)
87  backup_dir = ".MP_ali_list"
88  try:
89  os.makedirs(backup_dir)
90  except OSError as e:
91  if e.args == (17, 'File exists'):
92  pass
93  else:
94  raise
95  with open(os.path.join(backup_dir, campaign_list), "a") as f:
96  fcntl.flock(f, fcntl.LOCK_EX)
97  f.write(campaign_info)
98  fcntl.flock(f, fcntl.LOCK_UN)
99  print(" - updated campaign list '"+campaign_list+"'")
100 
101  if args.copy is None:
102  copy_default_templates(args, next_campaign)
103  else:
104  copied_files = []
105  for ext in ("py", "ini", "txt"):
106  for config_file in glob.glob(args.copy+"/*."+ext):
107  copied_files.append(os.path.basename(config_file))
108  shutil.copy(config_file, next_campaign)
109  if len(copied_files) == 0:
110  print(" - no configuration files for '"+args.copy+"'")
111  copy_default_templates(args, next_campaign)
112  else:
113  alignment_config_ini = os.path.join(next_campaign,
114  "alignment_config.ini")
115  regex_input = (r"^(jobname\s*[=:])(\s*)"+
116  os.path.basename(args.copy.strip("/"))+r"\s*$",
117  r"\1 "+next_campaign+r"\n")
118  if os.path.isfile(alignment_config_ini):
119  customize_default_template(alignment_config_ini,
120  regex_input)
121  print(" - copied configuration files from", end=' ')
122  print("'"+args.copy+"':", ", ".join(copied_files))
123 
124  except OSError as e:
125  if e.args == (17, 'File exists'):
126  next_number += 1 # someone created a campaign ~at the same time
127  continue
128  else:
129  raise
130  break
131 
132 

References add_campaign(), helper.checked_out_MPS(), copy_default_templates(), customize_default_template(), get_first_match(), join(), genParticles_cff.map, print(), and str.

Variable Documentation

◆ required_version

tuple mps_setup_new_align.required_version = (2,7)

Definition at line 20 of file mps_setup_new_align.py.

mps_setup_new_align.customize_default_template
def customize_default_template(file_name, *regex_replace_pairs)
Definition: mps_setup_new_align.py:227
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:17
mps_setup_new_align.copy_default_templates
def copy_default_templates(args, next_campaign)
Definition: mps_setup_new_align.py:198
mps_setup_new_align.add_campaign
def add_campaign(campaign_file, campaign, args)
Definition: mps_setup_new_align.py:151
mps_setup_new_align.main
def main(argv=None)
Definition: mps_setup_new_align.py:29
submitPVValidationJobs.split
def split(sequence, size)
Definition: submitPVValidationJobs.py:352
str
#define str(s)
Definition: TestProcessor.cc:52
mps_setup_new_align.get_first_match
def get_first_match(regex, directory)
Definition: mps_setup_new_align.py:134
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:46
createfilelist.int
int
Definition: createfilelist.py:10
helper.checked_out_MPS
def checked_out_MPS()
Definition: helper.py:5
format
genParticles_cff.map
map
Definition: genParticles_cff.py:11