CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
conddb_migrate Namespace Reference

Functions

def check_and_run
 
def fetch_gts
 
def main
 
def make_gt_connection
 
def migrate_account
 
def migrate_accounts
 
def migrate_gt
 
def migrate_gts
 
def run_command
 
def tags_in_gts
 

Variables

list accounts
 

Function Documentation

def conddb_migrate.check_and_run (   args)

Definition at line 180 of file conddb_migrate.py.

Referenced by main().

180 
181 def check_and_run(args):
182  if 'SCRAM_ARCH' not in os.environ:
183  raise Exception('SCRAM_ARCH needs to be set: run cmsenv within a newish release.')
184 
185  if 'CMSSW_BASE' not in os.environ:
186  raise Exception('CMSSW_BASE needs to be set: run cmsenv within a newish release.')
187 
188  if 'jobs' in args and args.jobs <= 0:
189  raise Exception('If set, --jobs needs to be >= 1.')
190 
191  aliases = {
192  'root': 'oracle://cms_orcoff_prep/CMS_R5_CONDITIONS',
193  'boost': 'oracle://cms_orcoff_prep/CMS_CONDITIONS',
194  }
195 
196  if args.db in aliases:
197  args.db = aliases[args.db]
198 
199  # Check that the release and database match to prevent mistakes...
200  if args.db == 'oracle://cms_orcoff_prep/CMS_CONDITIONS' and \
201  not 'BOOST' in os.environ['CMSSW_VERSION']:
202  raise Exception('Boost database without a Boost release -- mistake?')
203 
204  if args.db == 'oracle://cms_orcoff_prep/CMS_R5_CONDITIONS' and \
205  'BOOST' in os.environ['CMSSW_VERSION']:
206  raise Exception('ROOT database with a Boost release -- mistake?')
207 
208  # Create output log folder
209  os.makedirs(args.output)
210 
211  args.func(args)
212 
conddb_migrate.check_and_run
def check_and_run
Definition: conddb_migrate.py:180
def conddb_migrate.fetch_gts (   connection)

Definition at line 123 of file conddb_migrate.py.

References ComparisonHelper.zip().

Referenced by migrate_gts(), and tags_in_gts().

123 
124 def fetch_gts(connection):
125  logging.info('Fetching global tag list...')
126  cursor = connection.cursor()
127  cursor.execute('''
128  select substr(table_name, length('tagtree_table_') + 1) gt
129  from all_tables
130  where owner = 'CMS_COND_31X_GLOBALTAG'
131  and table_name like 'TAGTREE_TABLE_%'
132  order by gt
133  ''')
134  gts = zip(*cursor.fetchall())[0]
135  logging.info('Fetching global tag list... Done: %s global tags found.', len(gts))
136  return gts
137 
conddb_migrate.fetch_gts
def fetch_gts
Definition: conddb_migrate.py:123
ComparisonHelper::zip
OutputIterator zip(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp)
Definition: L1TStage2CaloLayer1.h:41
def conddb_migrate.main ( ) 
Entry point.

Definition at line 213 of file conddb_migrate.py.

References check_and_run().

213 
214 def main():
215  '''Entry point.
216  '''
217 
218  parser = argparse.ArgumentParser(description='conddb_migrate - the CMS Conditions DB migration script')
219  parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v prints debugging information of this tool, like tracebacks in case of errors.')
220  parser.add_argument('--output', '-o', default=time.strftime('%Y-%m-%d-%H-%M-%S'), help='Output folder. Default: {current_timestamp}, i.e. %(default)s')
221  parser.add_argument('db', help='Destination database. Aliases: "root" (CMS_CONDITIONS), "boost" (CMS_TEST_CONDITIONS), both in prep. *Make sure the database kind matches the code, i.e. use a BOOST IB when uploading to a Boost database; and a normal release when uploading to the ROOT database -- this script checks the CMSSW_VERSION when using the two official aliases in prep to prevent mistakes, but not for other databases.*')
222  parser_subparsers = parser.add_subparsers(title='Available subcommands')
223 
224  parser_account = parser_subparsers.add_parser('account', description='Migrates all (non-migrated) tags, IOVs and payloads, from an account.')
225  parser_account.add_argument('account', help='The account to migrate.')
226  parser_account.set_defaults(func=migrate_account)
227 
228  parser_accounts = parser_subparsers.add_parser('accounts', description='Migrates all accounts (see "account" command).')
229  parser_accounts.add_argument('--jobs', '-j', type=int, default=4, help='Number of jobs.')
230  parser_accounts.set_defaults(func=migrate_accounts)
231 
232  parser_gt = parser_subparsers.add_parser('gt', description='Migrates a single global tag.')
233  parser_gt.add_argument('gt', help='The global tag to migrate.')
234  parser_gt.set_defaults(func=migrate_gt)
235 
236  parser_gts = parser_subparsers.add_parser('gts', description='Migrates all global tags (see "gt" command).')
237  parser_gts.add_argument('authpath', help='Authentication path.')
238  parser_gts.add_argument('--jobs', '-j', type=int, default=4, help='Number of jobs.')
239  parser_gts.set_defaults(func=migrate_gts)
240 
241  parser_tags_in_gts = parser_subparsers.add_parser('tags_in_gts', description='Dumps the set of tags (including account name) which are in each global tag.')
242  parser_tags_in_gts.add_argument('authpath', help='Authentication path.')
243  parser_tags_in_gts.set_defaults(func=tags_in_gts)
244 
245  args = parser.parse_args()
246 
247  logging.basicConfig(
248  format = '[%(asctime)s] %(levelname)s: %(message)s',
249  level = logging.DEBUG if args.verbose >= 1 else logging.INFO,
250  )
251 
252  if args.verbose >= 1:
253  # Include the traceback
254  check_and_run(args)
255  else:
256  # Only one error line
257  try:
258  check_and_run(args)
259  except Exception as e:
260  logging.error(e)
261  sys.exit(1)
262 
conddb_migrate.main
def main
Definition: conddb_migrate.py:213
conddb_migrate.check_and_run
def check_and_run
Definition: conddb_migrate.py:180
def conddb_migrate.make_gt_connection (   args)

Definition at line 117 of file conddb_migrate.py.

References digitizers_cfi.strip.

Referenced by migrate_gts(), and tags_in_gts().

117 
118 def make_gt_connection(args):
119  logging.info('Fetching global tag list...')
120  password = subprocess.check_output('''cat %s | grep -F 'CMS_COND_31X_GLOBALTAG' -2 | tail -1 | cut -d'"' -f4''' % os.path.join(args.authpath, 'readOnlyProd.xml'), shell=True).strip()
121  return cx_Oracle.connect('CMS_COND_GENERAL_R', password, 'cms_orcon_adg')
122 
conddb_migrate.make_gt_connection
def make_gt_connection
Definition: conddb_migrate.py:117
digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:45
def conddb_migrate.migrate_account (   args)

Definition at line 96 of file conddb_migrate.py.

References run_command().

96 
97 def migrate_account(args):
98  command_template = '$CMSSW_BASE/bin/$SCRAM_ARCH/conddb_migrate -s oracle://cms_orcon_adg/%s -d %s'
99  command = command_template % (args.account, args.db)
100  run_command(command, os.path.join(args.output, args.account))
101 
conddb_migrate.migrate_account
def migrate_account
Definition: conddb_migrate.py:96
conddb_migrate.run_command
def run_command
Definition: conddb_migrate.py:87
def conddb_migrate.migrate_accounts (   args)

Definition at line 102 of file conddb_migrate.py.

102 
103 def migrate_accounts(args):
104  def _make_args(args, account):
105  newargs = argparse.Namespace(**vars(args))
106  newargs.account = account
107  return newargs
108 
109  multiprocessing.Pool(args.jobs).map(migrate_account, [_make_args(args, account) for account in accounts])
110 
conddb_migrate.migrate_accounts
def migrate_accounts
Definition: conddb_migrate.py:102
def conddb_migrate.migrate_gt (   args)

Definition at line 111 of file conddb_migrate.py.

References run_command().

111 
112 def migrate_gt(args):
113  command_template = '$CMSSW_BASE/bin/$SCRAM_ARCH/conddb_migrate_gt -s oracle://cms_orcon_adg/CMS_COND_31X_GLOBALTAG -d %s -g %s'
114  command = command_template % (args.db, args.gt)
115  run_command(command, os.path.join(args.output, args.gt))
116 
conddb_migrate.migrate_gt
def migrate_gt
Definition: conddb_migrate.py:111
conddb_migrate.run_command
def run_command
Definition: conddb_migrate.py:87
def conddb_migrate.migrate_gts (   args)

Definition at line 138 of file conddb_migrate.py.

References fetch_gts(), and make_gt_connection().

138 
139 def migrate_gts(args):
140  gts = fetch_gts(make_gt_connection(args))
141 
142  def _make_args(args, gt):
143  newargs = argparse.Namespace(**vars(args))
144  newargs.gt = gt
145  return newargs
146 
147  multiprocessing.Pool(args.jobs).map(migrate_gt, [_make_args(args, gt) for gt in gts])
148 
conddb_migrate.fetch_gts
def fetch_gts
Definition: conddb_migrate.py:123
conddb_migrate.make_gt_connection
def make_gt_connection
Definition: conddb_migrate.py:117
conddb_migrate.migrate_gts
def migrate_gts
Definition: conddb_migrate.py:138
def conddb_migrate.run_command (   command,
  output_file 
)

Definition at line 87 of file conddb_migrate.py.

Referenced by migrate_account(), and migrate_gt().

87 
88 def run_command(command, output_file):
89  command = '%s > %s 2>&1' % (command, output_file)
90  logging.info('Running %s', command)
91  try:
92  subprocess.check_call(command, shell=True)
93  except subprocess.CalledProcessError as e:
94  logging.error('Error while running %s: return code %s', command, e.returncode)
95 
conddb_migrate.run_command
def run_command
Definition: conddb_migrate.py:87
def conddb_migrate.tags_in_gts (   args)

Definition at line 149 of file conddb_migrate.py.

References PVValHelper.add(), fetch_gts(), make_gt_connection(), and print().

149 
150 def tags_in_gts(args):
151  # Dynamic SQL is used due to the schema
152  # This is OK since we trust the input,
153  # which is the GT database's tables.
154 
155  connection = make_gt_connection(args)
156  gts = fetch_gts(connection)
157  account_tags = {}
158 
159  for i, gt in enumerate(gts):
160  logging.info('[%s/%s] Reading %s ...', i+1, len(gts), gt)
161  cursor = connection.cursor()
162  cursor.execute('''
163  select "pfn", "tagname"
164  from CMS_COND_31X_GLOBALTAG.TAGINVENTORY_TABLE
165  where "tagid" in (
166  select "tagid"
167  from CMS_COND_31X_GLOBALTAG.TAGTREE_TABLE_%s
168  )
169  ''' % gt)
170 
171  for account, tag in cursor:
172  account_tags.setdefault(account, set([])).add(tag)
173 
174  for account in sorted(account_tags):
175  print(account)
176  for tag in sorted(account_tags[account]):
177  print(' ', tag)
178  print()
179 
conddb_migrate.fetch_gts
def fetch_gts
Definition: conddb_migrate.py:123
conddb_migrate.tags_in_gts
def tags_in_gts
Definition: conddb_migrate.py:149
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
PVValHelper::add
void add(std::map< std::string, TH1 * > &h, TH1 *hist)
Definition: PVValidationHelpers.cc:12
conddb_migrate.make_gt_connection
def make_gt_connection
Definition: conddb_migrate.py:117

Variable Documentation

list conddb_migrate.accounts

Definition at line 18 of file conddb_migrate.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5