confdbOfflineConverter.OfflineConverter Class Reference

Public Member Functions
def	__init__ (self, version='v3', database='run3', url=None, verbose=False, proxy=False, proxyHost='localhost', proxyPort='8080', tunnel=False, tunnelPort='10121')
def	query (self, args)

Static Public Member Functions
def	CheckTempDirectory (dir)

Public Attributes
	baseDir
	baseUrl
	connect
	jars
	javaCmd
	proxy
	proxy_connect_args
	proxyHost
	proxyPort
	tunnel
	tunnel_connect_args
	tunnelPort
	verbose
	version
	workDir

Static Public Attributes
	databases
	dbparams
	ips_for_proxy

Detailed Description

Definition at line 13 of file confdbOfflineConverter.py.

Constructor & Destructor Documentation

__init__()

def confdbOfflineConverter.OfflineConverter.__init__	(		self,
			version = 'v3',
			database = 'run3',
			url = None,
			verbose = False,
			proxy = False,
			proxyHost = 'localhost',
			proxyPort = '8080',
			tunnel = False,
			tunnelPort = '10121'
	)

Definition at line 76 of file confdbOfflineConverter.py.

Referenced by confdbOfflineConverter.OfflineConverter.CheckTempDirectory().

                 tunnel = False, tunnelPort = '10121'):
        self.verbose = verbose
        self.version = version
        self.baseDir = '/afs/cern.ch/user/c/confdb/www/%s/lib' % version
        self.baseUrl = 'https://confdb.web.cern.ch/confdb/%s/lib' % version
        self.jars    = ( 'ojdbc8.jar', 'cmssw-evf-confdb-converter.jar' )
        if version=='v2':
            #legacy driver for run2 gui
            self.jars = ( 'ojdbc6.jar', 'cmssw-evf-confdb-converter.jar' )
        self.workDir = ''
        self.proxy = proxy
        self.proxyHost = proxyHost
        self.proxyPort = proxyPort
        self.tunnel = tunnel
        self.tunnelPort = tunnelPort

        if self.proxy and self.tunnel:
            sys.stderr.write( "ERROR: proxy and tunnel options can not both be true" )
            sys.exit(1)

        # check the schema version
        if version not in self.databases:
            # unsupported database version
            sys.stderr.write( "ERROR: unsupported database version \"%s\"\n" % version)

        # check the database
        if database in self.databases[version]:
            # load the connection parameters for the given database
            self.connect = self.databases[version][database]
        else:
            # unsupported database
            sys.stderr.write( "ERROR: unknown database \"%s\" for version \"%s\"\n" % (database, version))
            sys.exit(1)

        if self.proxy:
            self.proxy_connect_args = ('--dbproxy', '--dbproxyport', self.proxyPort, '--dbproxyhost', self.proxyHost)
            temp_connect = []
            for entry in self.connect:
                for key,item in self.ips_for_proxy.items():
                    entry = entry.replace(key,item)
                temp_connect.append(entry.replace(key,item))
            self.connect  = tuple(temp_connect)
        else:
            self.proxy_connect_args = ()

        # this sets the host to localhost
        if self.tunnel:
            temp_connect = list(self.connect)
            host_index = temp_connect.index('-h')
            temp_connect[host_index+1] = "localhost"
            self.connect = tuple(temp_connect)
            self.tunnel_connect_args = ('--dbport', self.tunnelPort)
        else:
            self.tunnel_connect_args = ()

        # check for a custom base URL
        if url is not None:
            self.baseUrl = url

        # try to read the .jar files from AFS, or download them
        if os.path.isdir(self.baseDir) and all(os.path.isfile(self.baseDir + '/' + jar) for jar in self.jars):
            # read the .jar fles from AFS
            self.workDir = self.baseDir
        else:
            # try to use $CMSSW_BASE/tmp
            self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['CMSSW_BASE'],'tmp','confdb',self.version))
            if not self.workDir:
                # try to use $TMP
                self.workDir = OfflineConverter.CheckTempDirectory(os.path.join(os.environ['TMP'],'confdb',self.version))
            if not self.workDir:
                # create a new temporary directory, and install a cleanup callback
                self.workDir = tempfile.mkdtemp()
                atexit.register(shutil.rmtree, self.workDir)
            # download the .jar files
            version_website = requests.get(self.baseUrl+"/../confdb.version").text
            jars_require_update = True
            if os.path.exists(os.path.join(self.workDir,"confdb.version")):
                with open(os.path.join(self.workDir,"confdb.version")) as f:
                    version_existing = f.read()                
                    if version_existing==version_website:
                        jars_require_update = False

            if jars_require_update:
                for jar in self.jars:
                    # download to a temporay name and use an atomic rename (in case an other istance is downloading the same file
                    handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = jar + '.')
                    os.close(handle)
                    request = requests.get(self.baseUrl + '/' + jar)
                    with open(temp,'wb') as f:
                        f.write(request.content)
                    os.rename(temp, self.workDir + '/' + jar)
                #jars updated, write their version
                handle, temp = tempfile.mkstemp(dir = self.workDir, prefix = "confdb.version" + '.')
                os.close(handle)
                with open(temp,'w') as f:
                    f.write(version_website)
                os.rename(temp,os.path.join(self.workDir,"confdb.version"))

        # setup the java command line and CLASSPATH
        if self.verbose:
            sys.stderr.write("workDir = %s\n" % self.workDir)
        # use non-blocking random number source /dev/urandom (instead of /dev/random), see:
        #   http://blockdump.blogspot.fr/2012/07/connection-problems-inbound-connection.html
        # deal with timezone region not found
        #   http://stackoverflow.com/questions/9156379/ora-01882-timezone-region-not-found
        # increase the thread stack size from the default of 1 MB to work around java.lang.StackOverflowError errors, see
        #   man java
        self.javaCmd = ( 'java', '-cp', ':'.join(self.workDir + '/' + jar for jar in self.jars), '-Djava.security.egd=file:///dev/urandom', '-Doracle.jdbc.timezoneAsRegion=false', '-Xss32M', 'confdb.converter.BrowserConverter' )

Member Function Documentation

CheckTempDirectory()

def confdbOfflineConverter.OfflineConverter.CheckTempDirectory (   dir )

static

Definition at line 64 of file confdbOfflineConverter.py.

References confdbOfflineConverter.OfflineConverter.__init__().

    def CheckTempDirectory(dir):
        dir = os.path.realpath(dir)
        if not os.path.isdir(dir):
            try:
                os.makedirs(dir)
            except:
                return None
        return dir

query()

def confdbOfflineConverter.OfflineConverter.query	(		self,
			args
	)

Definition at line 186 of file confdbOfflineConverter.py.

References DDTOBRodAlgo.connect, confdbOfflineConverter.OfflineConverter.connect, confdbOfflineConverter.OfflineConverter.javaCmd, join(), confdbOfflineConverter.OfflineConverter.proxy_connect_args, confdbOfflineConverter.OfflineConverter.tunnel_connect_args, VisualizationOptions.verbose, L1TdeStage2EMTF.verbose, L1TStage2uGMTMuon.verbose, L1TStage2RegionalShower.verbose, L1TStage2EMTF.verbose, L1TdeStage2CPPF.verbose, L1TStage2CPPF.verbose, dqmTnP::AbstractFitter.verbose, DQMGenericTnPClient.verbose, L1TStage2OMTF.verbose, TritonService::FallbackOpts.verbose, DeviationsFromFileSensor2D.verbose, L1TStage2BMTF.verbose, L1TStage2MuonComp.verbose, confdbOfflineConverter.OfflineConverter.verbose, L1TStage2RegionalMuonCandComp.verbose, L1TCaloLayer1RawToDigi.verbose, HIPAlignmentAlgorithm.verbose, BeamSpotDipServer.verbose, o2olib.O2OJobMgr.verbose, L1TStage2uGTCaloLayer2Comp.verbose, OpticalObject.verbose, and L1TdeStage2CaloLayer2.verbose.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

    def query(self, *args):
        args = self.javaCmd + self.connect + self.proxy_connect_args + self.tunnel_connect_args + args 
        if self.verbose:
            sys.stderr.write("\n" + ' '.join(args) + "\n\n" )
        sub = subprocess.Popen(
            args,
            stdin = None,
            stdout = subprocess.PIPE,
            stderr = subprocess.PIPE,
            shell = False,
            universal_newlines = True )
        return sub.communicate()

Member Data Documentation

baseDir

confdbOfflineConverter.OfflineConverter.baseDir

Definition at line 79 of file confdbOfflineConverter.py.

baseUrl

confdbOfflineConverter.OfflineConverter.baseUrl

Definition at line 80 of file confdbOfflineConverter.py.

connect

confdbOfflineConverter.OfflineConverter.connect

Definition at line 104 of file confdbOfflineConverter.py.

Referenced by confdbOfflineConverter.OfflineConverter.query(), and o2o_db_cfgmap.DbManagerDAQ.update_hashmap().

databases

confdbOfflineConverter.OfflineConverter.databases

static

Definition at line 31 of file confdbOfflineConverter.py.

dbparams

confdbOfflineConverter.OfflineConverter.dbparams

static

Definition at line 59 of file confdbOfflineConverter.py.

ips_for_proxy

confdbOfflineConverter.OfflineConverter.ips_for_proxy

static

Definition at line 46 of file confdbOfflineConverter.py.

jars

confdbOfflineConverter.OfflineConverter.jars

Definition at line 81 of file confdbOfflineConverter.py.

javaCmd

confdbOfflineConverter.OfflineConverter.javaCmd

Definition at line 183 of file confdbOfflineConverter.py.

Referenced by confdbOfflineConverter.OfflineConverter.query().

proxy

confdbOfflineConverter.OfflineConverter.proxy

Definition at line 86 of file confdbOfflineConverter.py.

proxy_connect_args

confdbOfflineConverter.OfflineConverter.proxy_connect_args

Definition at line 111 of file confdbOfflineConverter.py.

Referenced by confdbOfflineConverter.OfflineConverter.query().

proxyHost

confdbOfflineConverter.OfflineConverter.proxyHost

Definition at line 87 of file confdbOfflineConverter.py.

proxyPort

confdbOfflineConverter.OfflineConverter.proxyPort

Definition at line 88 of file confdbOfflineConverter.py.

tunnel

confdbOfflineConverter.OfflineConverter.tunnel

Definition at line 89 of file confdbOfflineConverter.py.

tunnel_connect_args

confdbOfflineConverter.OfflineConverter.tunnel_connect_args

Definition at line 127 of file confdbOfflineConverter.py.

Referenced by confdbOfflineConverter.OfflineConverter.query().

tunnelPort

confdbOfflineConverter.OfflineConverter.tunnelPort

Definition at line 90 of file confdbOfflineConverter.py.

verbose

confdbOfflineConverter.OfflineConverter.verbose

Definition at line 77 of file confdbOfflineConverter.py.

Referenced by python.diff_provenance.difference.list_diff(), python.diffProv.difference.list_diff(), python.diff_provenance.difference.module_diff(), python.diffProv.difference.module_diff(), python.diff_provenance.difference.onefilemodules(), python.diffProv.difference.onefilemodules(), core.TriggerMatchAnalyzer.TriggerMatchAnalyzer.process(), core.SkimAnalyzerCount.SkimAnalyzerCount.process(), objects.VertexAnalyzer.VertexAnalyzer.process(), and confdbOfflineConverter.OfflineConverter.query().

version

confdbOfflineConverter.OfflineConverter.version

Definition at line 78 of file confdbOfflineConverter.py.

Referenced by validation.Sample.datasetpattern(), and validation.Sample.filename().

workDir

confdbOfflineConverter.OfflineConverter.workDir

Definition at line 85 of file confdbOfflineConverter.py.