CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
conddb_migrate Namespace Reference

Functions

def check_and_run
 
def fetch_gts
 
def main
 
def make_gt_connection
 
def migrate_account
 
def migrate_accounts
 
def migrate_gt
 
def migrate_gts
 
def run_command
 
def tags_in_gts
 

Variables

list accounts
 

Function Documentation

def conddb_migrate.check_and_run (   args)

Definition at line 179 of file conddb_migrate.py.

Referenced by main().

180 def check_and_run(args):
181  if 'SCRAM_ARCH' not in os.environ:
182  raise Exception('SCRAM_ARCH needs to be set: run cmsenv within a newish release.')
183 
184  if 'CMSSW_BASE' not in os.environ:
185  raise Exception('CMSSW_BASE needs to be set: run cmsenv within a newish release.')
186 
187  if 'jobs' in args and args.jobs <= 0:
188  raise Exception('If set, --jobs needs to be >= 1.')
189 
190  aliases = {
191  'root': 'oracle://cms_orcoff_prep/CMS_R5_CONDITIONS',
192  'boost': 'oracle://cms_orcoff_prep/CMS_CONDITIONS',
193  }
194 
195  if args.db in aliases:
196  args.db = aliases[args.db]
197 
198  # Check that the release and database match to prevent mistakes...
199  if args.db == 'oracle://cms_orcoff_prep/CMS_CONDITIONS' and \
200  not 'BOOST' in os.environ['CMSSW_VERSION']:
201  raise Exception('Boost database without a Boost release -- mistake?')
202 
203  if args.db == 'oracle://cms_orcoff_prep/CMS_R5_CONDITIONS' and \
204  'BOOST' in os.environ['CMSSW_VERSION']:
205  raise Exception('ROOT database with a Boost release -- mistake?')
206 
207  # Create output log folder
208  os.makedirs(args.output)
209 
210  args.func(args)
211 
def conddb_migrate.fetch_gts (   connection)

Definition at line 122 of file conddb_migrate.py.

Referenced by migrate_gts(), and tags_in_gts().

123 def fetch_gts(connection):
124  logging.info('Fetching global tag list...')
125  cursor = connection.cursor()
126  cursor.execute('''
127  select substr(table_name, length('tagtree_table_') + 1) gt
128  from all_tables
129  where owner = 'CMS_COND_31X_GLOBALTAG'
130  and table_name like 'TAGTREE_TABLE_%'
131  order by gt
132  ''')
133  gts = zip(*cursor.fetchall())[0]
134  logging.info('Fetching global tag list... Done: %s global tags found.', len(gts))
135  return gts
136 
def conddb_migrate.main ( )
Entry point.

Definition at line 212 of file conddb_migrate.py.

References check_and_run().

213 def main():
214  '''Entry point.
215  '''
216 
217  parser = argparse.ArgumentParser(description='conddb_migrate - the CMS Conditions DB migration script')
218  parser.add_argument('--verbose', '-v', action='count', help='Verbosity level. -v prints debugging information of this tool, like tracebacks in case of errors.')
219  parser.add_argument('--output', '-o', default=time.strftime('%Y-%m-%d-%H-%M-%S'), help='Output folder. Default: {current_timestamp}, i.e. %(default)s')
220  parser.add_argument('db', help='Destination database. Aliases: "root" (CMS_CONDITIONS), "boost" (CMS_TEST_CONDITIONS), both in prep. *Make sure the database kind matches the code, i.e. use a BOOST IB when uploading to a Boost database; and a normal release when uploading to the ROOT database -- this script checks the CMSSW_VERSION when using the two official aliases in prep to prevent mistakes, but not for other databases.*')
221  parser_subparsers = parser.add_subparsers(title='Available subcommands')
222 
223  parser_account = parser_subparsers.add_parser('account', description='Migrates all (non-migrated) tags, IOVs and payloads, from an account.')
224  parser_account.add_argument('account', help='The account to migrate.')
225  parser_account.set_defaults(func=migrate_account)
226 
227  parser_accounts = parser_subparsers.add_parser('accounts', description='Migrates all accounts (see "account" command).')
228  parser_accounts.add_argument('--jobs', '-j', type=int, default=4, help='Number of jobs.')
229  parser_accounts.set_defaults(func=migrate_accounts)
230 
231  parser_gt = parser_subparsers.add_parser('gt', description='Migrates a single global tag.')
232  parser_gt.add_argument('gt', help='The global tag to migrate.')
233  parser_gt.set_defaults(func=migrate_gt)
234 
235  parser_gts = parser_subparsers.add_parser('gts', description='Migrates all global tags (see "gt" command).')
236  parser_gts.add_argument('authpath', help='Authentication path.')
237  parser_gts.add_argument('--jobs', '-j', type=int, default=4, help='Number of jobs.')
238  parser_gts.set_defaults(func=migrate_gts)
239 
240  parser_tags_in_gts = parser_subparsers.add_parser('tags_in_gts', description='Dumps the set of tags (including account name) which are in each global tag.')
241  parser_tags_in_gts.add_argument('authpath', help='Authentication path.')
242  parser_tags_in_gts.set_defaults(func=tags_in_gts)
243 
244  args = parser.parse_args()
245 
246  logging.basicConfig(
247  format = '[%(asctime)s] %(levelname)s: %(message)s',
248  level = logging.DEBUG if args.verbose >= 1 else logging.INFO,
249  )
250 
251  if args.verbose >= 1:
252  # Include the traceback
253  check_and_run(args)
254  else:
255  # Only one error line
256  try:
257  check_and_run(args)
258  except Exception as e:
259  logging.error(e)
260  sys.exit(1)
261 
def conddb_migrate.make_gt_connection (   args)

Definition at line 116 of file conddb_migrate.py.

Referenced by migrate_gts(), and tags_in_gts().

117 def make_gt_connection(args):
118  logging.info('Fetching global tag list...')
119  password = subprocess.check_output('''cat %s | grep -F 'CMS_COND_31X_GLOBALTAG' -2 | tail -1 | cut -d'"' -f4''' % os.path.join(args.authpath, 'readOnlyProd.xml'), shell=True).strip()
120  return cx_Oracle.connect('CMS_COND_GENERAL_R', password, 'cms_orcon_adg')
121 
def conddb_migrate.migrate_account (   args)

Definition at line 95 of file conddb_migrate.py.

References run_command().

95 
96 def migrate_account(args):
97  command_template = '$CMSSW_BASE/bin/$SCRAM_ARCH/conddb_migrate -s oracle://cms_orcon_adg/%s -d %s'
98  command = command_template % (args.account, args.db)
99  run_command(command, os.path.join(args.output, args.account))
100 
def conddb_migrate.migrate_accounts (   args)

Definition at line 101 of file conddb_migrate.py.

References python.multivaluedict.map().

102 def migrate_accounts(args):
103  def _make_args(args, account):
104  newargs = argparse.Namespace(**vars(args))
105  newargs.account = account
106  return newargs
107 
108  multiprocessing.Pool(args.jobs).map(migrate_account, [_make_args(args, account) for account in accounts])
109 
def conddb_migrate.migrate_gt (   args)

Definition at line 110 of file conddb_migrate.py.

References run_command().

111 def migrate_gt(args):
112  command_template = '$CMSSW_BASE/bin/$SCRAM_ARCH/conddb_migrate_gt -s oracle://cms_orcon_adg/CMS_COND_31X_GLOBALTAG -d %s -g %s'
113  command = command_template % (args.db, args.gt)
114  run_command(command, os.path.join(args.output, args.gt))
115 
def conddb_migrate.migrate_gts (   args)

Definition at line 137 of file conddb_migrate.py.

References fetch_gts(), make_gt_connection(), and python.multivaluedict.map().

138 def migrate_gts(args):
139  gts = fetch_gts(make_gt_connection(args))
140 
141  def _make_args(args, gt):
142  newargs = argparse.Namespace(**vars(args))
143  newargs.gt = gt
144  return newargs
145 
146  multiprocessing.Pool(args.jobs).map(migrate_gt, [_make_args(args, gt) for gt in gts])
147 
def conddb_migrate.run_command (   command,
  output_file 
)

Definition at line 86 of file conddb_migrate.py.

Referenced by migrate_account(), and migrate_gt().

86 
87 def run_command(command, output_file):
88  command = '%s > %s 2>&1' % (command, output_file)
89  logging.info('Running %s', command)
90  try:
91  subprocess.check_call(command, shell=True)
92  except subprocess.CalledProcessError as e:
93  logging.error('Error while running %s: return code %s', command, e.returncode)
94 
def conddb_migrate.tags_in_gts (   args)

Definition at line 148 of file conddb_migrate.py.

References Clusterizer1DCommons.add(), fetch_gts(), and make_gt_connection().

149 def tags_in_gts(args):
150  # Dynamic SQL is used due to the schema
151  # This is OK since we trust the input,
152  # which is the GT database's tables.
153 
154  connection = make_gt_connection(args)
155  gts = fetch_gts(connection)
156  account_tags = {}
157 
158  for i, gt in enumerate(gts):
159  logging.info('[%s/%s] Reading %s ...', i+1, len(gts), gt)
160  cursor = connection.cursor()
161  cursor.execute('''
162  select "pfn", "tagname"
163  from CMS_COND_31X_GLOBALTAG.TAGINVENTORY_TABLE
164  where "tagid" in (
165  select "tagid"
166  from CMS_COND_31X_GLOBALTAG.TAGTREE_TABLE_%s
167  )
168  ''' % gt)
169 
170  for account, tag in cursor:
171  account_tags.setdefault(account, set([])).add(tag)
172 
173  for account in sorted(account_tags):
174  print account
175  for tag in sorted(account_tags[account]):
176  print ' ', tag
177  print
178 
void add(const std::vector< const T * > &source, std::vector< const T * > &dest)

Variable Documentation

list conddb_migrate.accounts

Definition at line 17 of file conddb_migrate.py.