Reference Manual

def uploadConditions_v2.parse_arguments ( )

Definition at line 233 of file uploadConditions_v2.py.

References beamvalidation.exit(), getInput(), join(), print(), runWizard(), and str.

 
 def parse_arguments():
     # read in command line arguments, and build metadata dictionary from them
     parser = optparse.OptionParser(description="CMS Conditions Upload Script in CondDBFW.",
         usage = 'Usage: %prog [options] <file>')
 
     # metadata arguments
     parser.add_option("-i", "--inputTag", type=str,\
                         help="Tag to take IOVs + Payloads from in --sourceDB.")
     parser.add_option("-t", "--destinationTag", type=str,\
                         help="Tag to copy IOVs + Payloads to in --destDB.")
     parser.add_option("-D", "--destinationDatabase", type=str,\
                         help="Database to copy IOVs + Payloads to.")
     parser.add_option("-s", "--since", type=int,\
                         help="Since to take IOVs from.")
     parser.add_option("-u", "--userText", type=str,\
                         help="Description of --destTag (can be empty).")
 
     # non-metadata arguments
     parser.add_option("-m", "--metadataFile", type=str, help="Metadata file to take metadata from.")
 
     parser.add_option("-d", "--debug", action="store_true", default=False)
     parser.add_option("-v", "--verbose", action="store_true", default=False)
     parser.add_option("-T", "--testing", action="store_true")
     parser.add_option("--fcsr-filter", type=str, help="Synchronization to take FCSR from for local filtering of IOVs.")
 
     parser.add_option("-n", "--netrc", help = 'The netrc host (machine) from where the username and password will be read.')
     
     parser.add_option("-a", "--authPath", help = 'The path of the .netrc file for the authentication. Default: $HOME')
 
     parser.add_option("-H", "--hashToUse")
 
     parser.add_option("-S", "--server")
 
     parser.add_option("-o", "--review-options", action="store_true")
 
     parser.add_option("-r", "--replay-file")
 
     (command_line_data, arguments) = parser.parse_args()
 
     if len(arguments) < 1:
         if command_line_data.hashToUse == None:
             parser.print_help()
             exit(-2)
     
     command_line_data.sourceDB = arguments[0]
 
     if command_line_data.replay_file:
         dictionary = json.loads("".join(open(command_line_data.replay_file, "r").readlines()))
         command_line_data.tier0_response = dictionary["tier0_response"]
 
     # default is the production server, which can point to either database anyway
     server_alias_to_url = {
         "prep" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/",
         "dev" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/",
         "prod" : "https://cms-conddb.cern.ch/cmsDbCondUpload/"
     }
 
     # if prep, prod or None were given, convert to URLs in dictionary server_alias_to_url
     # if not, assume a URL has been given and use this instead
     if command_line_data.server in server_alias_to_url.keys():
         command_line_data.server = server_alias_to_url[command_line_data.server]
 
     # resolve destination databases
     database_alias_to_connection = {
         "prep": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
         "dev": "oracle://cms_orcoff_prep/CMS_CONDITIONS",
         "prod": "oracle://cms_orcon_adg/CMS_CONDITIONS"
     }
     
     if command_line_data.destinationDatabase in database_alias_to_connection.keys():
         command_line_data.destinationDatabase = database_alias_to_connection[command_line_data.destinationDatabase]
 
 
     # use netrc to get username and password
     try:
         netrc_file = command_line_data.netrc
         auth_path = command_line_data.authPath
         if not auth_path is None:
             if netrc_file is None:
                 netrc_file = os.path.join(auth_path,'.netrc')
             else:
                 netrc_file = os.path.join(auth_path, netrc_file)
 
         netrc_authenticators = netrc.netrc(netrc_file).authenticators("ConditionUploader")
         if netrc_authenticators == None:
             print("Your netrc file must contain the key 'ConditionUploader'.")
             manual_input = raw_input("Do you want to try to type your credentials? ")
             if manual_input == "y":
                 # ask for username and password
                 username = raw_input("Username: ")
                 password = getpass.getpass("Password: ")
             else:
                 exit()
         else:
             username = netrc_authenticators[0]
             password = netrc_authenticators[2]
     except:
         print("Couldn't obtain your credentials (either from netrc or manual input).")
         exit()
 
     command_line_data.username = username
     command_line_data.password = password
     # this will be used as the final destinationTags value by all input methods
     # apart from the metadata file
     command_line_data.destinationTags = {command_line_data.destinationTag:{}}
 
     """
     Construct metadata_dictionary:
     Currently, this is 3 cases:
 
     1) An IOV is being appended to an existing Tag with an existing Payload.
     In this case, we just take all data from the command line.
 
     2) No metadata file is given, so we assume that ALL upload metadata is coming from the command line.
 
     3) A metadata file is given, hence we parse the file, and then iterate through command line arguments
     since these override the options set in the metadata file.
 
     """
 
     # Hash to use, entirely from command line
     if command_line_data.hashToUse != None:
         command_line_data.userText = ""
         metadata_dictionary = command_line_data.__dict__
     elif command_line_data.metadataFile == None:
         if command_line_data.sourceDB != None and (command_line_data.inputTag == None or command_line_data.destinationTag == None or command_line_data.destinationDatabase == None):
             basepath = command_line_data.sourceDB.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
             basename = os.path.basename(basepath)
             dataFilename = '%s.db' % basepath
             metadataFilename = '%s.txt' % basepath
 
             # Data file
             try:
                 with open(dataFilename, 'rb') as dataFile:
                     pass
             except IOError as e:
                 errMsg = 'Impossible to open SQLite data file %s' %dataFilename
                 print( errMsg )
                 ret['status'] = -3
                 ret['error'] = errMsg
                 return ret
 
             # Metadata file
 
             command_line_data.sourceDB = dataFilename
 
             try:
                 with open(metadataFilename, 'rb') as metadataFile:
                     pass
             except IOError as e:
                 if e.errno != errno.ENOENT:
                     errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename
                     ret = {}
                     ret['status'] = -4
                     ret['error'] = errMsg
                     exit (ret)
 
                 if getInput('y', '\nIt looks like the metadata file %s does not exist and not enough parameters were received in the command line. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y':
                     errMsg = 'Metadata file %s does not exist' %metadataFilename
                     ret = {}
                     ret['status'] = -5
                     ret['error'] = errMsg
                     exit(ret)
                 # Wizard
                 runWizard(basename, dataFilename, metadataFilename)
             command_line_data.metadataFile = metadataFilename
         else:
             command_line_data.userText = command_line_data.userText\
                                         if command_line_data.userText != None\
                                         else str(raw_input("Tag's description [can be empty]:"))
             metadata_dictionary = command_line_data.__dict__
 
     if command_line_data.metadataFile != None:
         metadata_dictionary = json.loads("".join(open(os.path.abspath(command_line_data.metadataFile), "r").readlines()))
         metadata_dictionary["username"] = username
         metadata_dictionary["password"] = password
         metadata_dictionary["userText"] = metadata_dictionary.get("userText")\
                                             if metadata_dictionary.get("userText") != None\
                                             else str(raw_input("Tag's description [can be empty]:"))
 
         # go through command line options and, if they are set, overwrite entries
         for (option_name, option_value) in command_line_data.__dict__.items():
             # if the metadata_dictionary sets this, overwrite it
             if option_name != "destinationTags":
                 if option_value != None or (option_value == None and not(option_name in metadata_dictionary.keys())):
                     # if option_value has a value, override the metadata file entry
                     # or if option_value is None but the metadata file doesn't give a value,
                     # set the entry to None as well
                     metadata_dictionary[option_name] = option_value
             else:
                 if option_value != {None:{}}:
                     metadata_dictionary["destinationTags"] = {option_value:{}}
                 elif option_value == {None:{}} and not("destinationTags" in metadata_dictionary.keys()):
                     metadata_dictionary["destinationTags"] = {None:{}}
 
     if command_line_data.review_options:
         defaults = {
             "since" : "Since of first IOV",
             "userText" : "Populated by upload process",
             "netrc" : "None given",
             "fcsr_filter" : "Don't apply",
             "hashToUse" : "Using local SQLite file instead"
         }
         print("Configuration to use for the upload:")
         for key in metadata_dictionary:
             if not(key) in ["username", "password", "destinationTag"]:
                 value_to_print = metadata_dictionary[key] if metadata_dictionary[key] != None else defaults[key]
                 print("\t%s : %s" % (key, value_to_print))
 
         if raw_input("\nDo you want to continue? [y/n] ") != "y":
             exit()
 
     if metadata_dictionary["server"] == None:
         if metadata_dictionary["destinationDatabase"] == "oracle://cms_orcoff_prep/CMS_CONDITIONS":
             metadata_dictionary["server"] = server_alias_to_url["prep"]
         else:
             metadata_dictionary["server"] = server_alias_to_url["prod"]
 
     return metadata_dictionary

uploadConditions_v2.runWizard

def runWizard

Definition: uploadConditions_v2.py:112

uploadConditions_v2.getInput

def getInput

Definition: uploadConditions_v2.py:62

void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)

Definition: Utilities.cc:47

join

static std::string join(char **cmd)

Definition: RemoteFile.cc:19

beamvalidation.exit

def exit

Definition: beamvalidation.py:52

uploadConditions_v2.parse_arguments

def parse_arguments

Definition: uploadConditions_v2.py:233

str

#define str(s)

Definition: TestProcessor.cc:53

def uploadConditions_v2.runWizard	(	basename,
		dataFilename,
		metadataFilename
	)

Definition at line 112 of file uploadConditions_v2.py.

References getInput(), getInputChoose(), getInputRepeat(), print(), and ComparisonHelper.zip().

Referenced by parse_arguments().

 
 def runWizard(basename, dataFilename, metadataFilename):
     while True:
         print('''\nWizard for metadata for %s
 
 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename)
 
         # Try to get the available inputTags
         try:
             dataConnection = sqlite3.connect(dataFilename)
             dataCursor = dataConnection.cursor()
             dataCursor.execute('select name from sqlite_master where type == "table"')
             tables = set(zip(*dataCursor.fetchall())[0])
 
             # only conddb V2 supported...
             if 'TAG' in tables:
                 dataCursor.execute('select NAME from TAG')
             # In any other case, do not try to get the inputTags
             else:
                 raise Exception()
 
             inputTags = dataCursor.fetchall()
             if len(inputTags) == 0:
                 raise Exception()
             inputTags = list(zip(*inputTags))[0]
 
         except Exception:
             inputTags = []
 
         if len(inputTags) == 0:
             print('\nI could not find any input tag in your data file, but you can still specify one manually.')
 
             inputTag = getInputRepeat(
                 '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ')
 
         else:
             print('\nI found the following input tags in your SQLite data file:')
             for (index, inputTag) in enumerate(inputTags):
                 print('   %s) %s' % (index, inputTag))
 
             inputTag = getInputChoose(inputTags, '0',
                                       '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
 
         databases = {
             'oraprod': 'oracle://cms_orcon_prod/CMS_CONDITIONS',
             'prod': 'oracle://cms_orcon_prod/CMS_CONDITIONS',
             'oradev': 'oracle://cms_orcoff_prep/CMS_CONDITIONS',
             'prep': 'oracle://cms_orcoff_prep/CMS_CONDITIONS',
         }
 
         destinationDatabase = ''
         ntry = 0
         print('\nWhich is the destination database where the tags should be exported?')
         print('\n%s) %s' % ('oraprod', databases['oraprod']))
         print('\n%s) %s' % ('oradev', databases['oradev']))
             
         while ( destinationDatabase not in databases.values() ): 
             if ntry==0:
                 inputMessage = \
                 '\nPossible choices: oraprod or oradev \ndestinationDatabase: '
             elif ntry==1:
                 inputMessage = \
                 '\nPlease choose one of the two valid destinations: oraprod or oradev \ndestinationDatabase: '
             else:
                 raise Exception('No valid destination chosen. Bailing out...')
             
             databaseInput = getInputRepeat(inputMessage).lower()
             if databaseInput in databases.keys():
                 destinationDatabase = databases[databaseInput]
             ntry += 1
 
         while True:
             since = getInput('',
                              '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
             if not since:
                 since = None
                 break
             else:
                 try:
                     since = int(since)
                     break
                 except ValueError:
                     print('The since value has to be an integer or empty (null).')
 
         userText = getInput('',
                             '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
 
         destinationTags = {}
         while True:
             destinationTag = getInput('',
                                       '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
             if not destinationTag:
                 if len(destinationTags) == 0:
                     print('There must be at least one destination tag.')
                     continue
                 break
 
             if destinationTag in destinationTags:
                 print(
                     'You already added this destination tag. Overwriting the previous one with this new one.')
 
             destinationTags[destinationTag] = {
             }
 
         metadata = {
             'destinationDatabase': destinationDatabase,
             'destinationTags': destinationTags,
             'inputTag': inputTag,
             'since': since,
             'userText': userText,
         }
 
         metadata = json.dumps(metadata, sort_keys=True, indent=4)
         print('\nThis is the generated metadata:\n%s' % metadata)
 
         if getInput('n',
                     '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
             break
     print('Saving generated metadata in %s...'% metadataFilename)
     with open(metadataFilename, 'wb') as metadataFile:
         metadataFile.write(metadata)

uploadConditions_v2.runWizard

def runWizard

Definition: uploadConditions_v2.py:112

uploadConditions_v2.getInput

def getInput

Definition: uploadConditions_v2.py:62

ComparisonHelper::zip

OutputIterator zip(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp)

Definition: L1TStage2CaloLayer1.h:41

void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)

Definition: Utilities.cc:47

uploadConditions_v2.getInputChoose

def getInputChoose

Definition: uploadConditions_v2.py:86

uploadConditions_v2.getInputRepeat

def getInputRepeat

Definition: uploadConditions_v2.py:101

Exception

Variables
int	__version__ = 1

list	final_service_url = upload_metadata["server"]

string	horizontal_rule = "="

tuple	response = get_version(final_service_url)

tuple	server_version = json.loads(response)

tuple	upload_metadata = parse_arguments()

dictionary	upload_metadata_argument = {}

def uploadConditions_v2.getInputChoose	(	optionsList,
		default,
		prompt = `''`
	)

Functions

Variables

Function Documentation

Variable Documentation

Functions
def	get_version

def	getInput

def	getInputChoose

def	getInputRepeat

def	getInputWorkflow

def	parse_arguments

def	run_upload

def	runWizard