test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Static Public Member Functions | Public Attributes | Static Public Attributes | Private Member Functions
querying.connection Class Reference

Public Member Functions

def __init__
 
def __repr__
 
def build_frontier_url
 
def build_oracle_url
 
def close_session
 
def commit
 
def deepcopy_model
 
def global_tag
 
def global_tag_map
 
def global_tag_map_request
 
def iov
 
def model
 
def object
 
def payload
 
def record
 
def search_everything
 
def setup
 
def tag
 
def write
 
def write_and_commit
 

Static Public Member Functions

def class_name_to_column
 

Public Attributes

 db_name
 
 factory
 
 models
 
 schema
 
 sessionmaker
 

Static Public Attributes

 base = None
 
 connection = None
 
 connection_data = None
 
 engine = None
 
 netrc_authenticators = None
 
int row_limit = 1000
 
 secrets = None
 
 session = None
 

Private Member Functions

def _cms_frontier_string
 
def _cms_oracle_string
 
def _get_CMS_frontier_connection_string
 
def _oracle_match_format
 

Detailed Description

Definition at line 15 of file querying.py.

Constructor & Destructor Documentation

def querying.connection.__init__ (   self,
  connection_data 
) 
try:
    import cx_Oracle
except ImportError as e:
    exit("cx_Oracle cannot be imported - try to run 'source /data/cmssw/setupEnv.sh' and 'source venv/bin/activate'.")

Definition at line 27 of file querying.py.

References querying.connection.connection_data, and split.

27 
28  def __init__(self, connection_data):
29  # is not needed in cmssw
30  """try:
31  import cx_Oracle
32  except ImportError as e:
33  exit("cx_Oracle cannot be imported - try to run 'source /data/cmssw/setupEnv.sh' and 'source venv/bin/activate'.")"""
34 
35  # todo translation on connection_data - it may be a string
36  # find out which formats of db string are acceptable
37  frontier_str_length = len("frontier://")
38  sqlite_str_length = len("sqlite:///")
39  if type(connection_data) == str and connection_data[0:frontier_str_length] == "frontier://":
40  db_name = connection_data[frontier_str_length:].split("/")[0]
41  schema = connection_data[frontier_str_length:].split("/")[1]
42  connection_data = {}
43  connection_data["db_alias"] = db_name
44  connection_data["schema"] = schema
45  connection_data["host"] = "frontier"
46  """elif type(connection_data) == str and connection_data[0:sqlite_str_length] == "sqlite:///":
47  db_name = connection_data[frontier_str_length:]
48  schema = ""
49  connection_data = {}
50  connection_data["db_alias"] = db_name
51  connection_data["schema"] = schema
52  connection_data["host"] = "sqlite"
53  """
54 
55  headers = ["login", "account", "password"]
56  self.connection_data = connection_data
57 
58  try:
59  self.schema = connection_data["schema"]
60  except KeyError as k:
61  self.schema = ""
62 
63  # setup authentication
64  import netrc
65  if connection_data["host"] == "oracle":
66  self.secrets = dict(zip(headers, netrc.netrc(connection_data["secrets"]).authenticators(connection_data["host"])))
67  self.netrc_authenticators = netrc.netrc(connection_data["secrets"])
68 
69  import models as ms
70  self.models = ms.generate()
71  self.base = self.models["Base"]
ComparisonHelper::zip
OutputIterator zip(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp)
Definition: L1TStage2CaloLayer1.h:38
split
double split
Definition: MVATrainer.cc:139

Member Function Documentation

def querying.connection.__repr__ (   self)

Definition at line 194 of file querying.py.

References querying.connection.db_name.

194 
195  def __repr__(self):
196  return "<connection db='%s'>" % self.db_name
def querying.connection._cms_frontier_string (   self,
  database,
  schema = "cms_conditions" 
)
private

Definition at line 135 of file querying.py.

References querying.connection._get_CMS_frontier_connection_string().

Referenced by querying.connection.build_frontier_url().

135 
136  def _cms_frontier_string(self, database, schema="cms_conditions"):
137  import urllib
138  return 'oracle+frontier://@%s/%s' % (urllib.quote_plus(self._get_CMS_frontier_connection_string(database)), schema)
querying.connection._get_CMS_frontier_connection_string
def _get_CMS_frontier_connection_string
Definition: querying.py:127
def querying.connection._cms_oracle_string (   self,
  user,
  pwd,
  db_name 
)
private

Definition at line 140 of file querying.py.

Referenced by querying.connection.build_oracle_url().

140 
141  def _cms_oracle_string(self, user, pwd, db_name):
142  return 'oracle://%s:%s@%s' % (user, pwd, db_name)
def querying.connection._get_CMS_frontier_connection_string (   self,
  database 
)
private

Definition at line 127 of file querying.py.

References communicate(), and cmsRelvalreport.exit.

Referenced by querying.connection._cms_frontier_string().

127 
128  def _get_CMS_frontier_connection_string(self, database):
129  try:
130  import subprocess
131  return subprocess.Popen(['cmsGetFnConnect', 'frontier://%s' % database], stdout = subprocess.PIPE).communicate()[0].strip()
132  except:
133  exit("Frontier connections can only be constructed when inside a CMSSW environment.")
querying.connection._get_CMS_frontier_connection_string
def _get_CMS_frontier_connection_string
Definition: querying.py:127
communicate
static void * communicate(void *obj)
Definition: DQMNet.cc:1246
def querying.connection._oracle_match_format (   self,
  string 
)
private

Definition at line 252 of file querying.py.

Referenced by querying.connection.search_everything().

252 
253  def _oracle_match_format(self, string):
254  return "%%%s%%" % string
def querying.connection.build_frontier_url (   self,
  db_name,
  schema 
)

Definition at line 173 of file querying.py.

References querying.connection._cms_frontier_string(), and reco.print().

173 
174  def build_frontier_url(self, db_name, schema):
175 
176  mapping = {
177  'pro': lambda: self._cms_frontier_string('PromptProd', schema),
178  'arc': lambda: self._cms_frontier_string('FrontierArc', schema),
179  'int': lambda: self._cms_frontier_string('FrontierInt', schema),
180  'dev': lambda: self._cms_frontier_string('FrontierPrep', schema)
181  }
182 
183  if db_name in mapping.keys():
184  database_url = mapping[db_name]()
185  else:
186  print("Database name given isn't valid.")
187  return
188 
189  try:
190  url = sqlalchemy.engine.url.make_url(database_url)
191  except sqlalchemy.exc.ArgumentError:
192  url = sqlalchemy.engine.url.make_url('sqlite:///%s' % db_name)
193  return url
reco::print
std::string print(const Track &, edm::Verbosity=edm::Concise)
Track print utility.
Definition: print.cc:10
def querying.connection.build_oracle_url (   self,
  user,
  pwd,
  db_name 
)

Definition at line 144 of file querying.py.

References querying.connection._cms_oracle_string(), and reco.print().

144 
145  def build_oracle_url(self, user, pwd, db_name):
146  # map db_name to the connection url
147  # pretty much the same as in conddblib.py in cmssw
148  mapping = {
149  'orapro': (lambda: self._cms_oracle_string(user, pwd, 'cms_orcon_adg')),
150  'oraarc': (lambda: self._cms_oracle_string(user, pwd, 'cmsarc_lb')),
151  'oraint': (lambda: self._cms_oracle_string(user, pwd, 'cms_orcoff_int')),
152  'oradev': (lambda: self._cms_oracle_string('cms_conditions_002', pwd, 'cms_orcoff_prep')),
153  'oraboost': (lambda: self._cms_oracle_string('cms_conditions', pwd, 'cms_orcon_adg')),
154  'oraboostprep': (lambda: self._cms_oracle_string('cms_conditions_002', pwd, 'cms_orcoff_prep')),
155 
156  'onlineorapro': (lambda: self._cms_oracle_string(user, pwd, 'cms_orcon_prod')),
157  'onlineoraint': (lambda: self._cms_oracle_string(user, pwd, 'cmsintr_lb')),
158  }
159 
160  if db_name in mapping.keys():
161  database_url = mapping[db_name]()
162  else:
163  print("Database name given isn't valid.")
164  return
165 
166  try:
167  url = sqlalchemy.engine.url.make_url(database_url)
168  if url.password is None:
169  url.password = pwd
170  except sqlalchemy.exc.ArgumentError:
171  url = sqlalchemy.engine.url.make_url('sqlite:///%s' % db_name)
172  return url
reco::print
std::string print(const Track &, edm::Verbosity=edm::Concise)
Track print utility.
Definition: print.cc:10
def querying.connection.class_name_to_column (   cls)
static

Definition at line 198 of file querying.py.

References join().

198 
199  def class_name_to_column(cls):
200  class_name = cls.__name__
201  all_upper_case = True
202  for character in class_name:
203  all_upper_case = character.isupper()
204  if all_upper_case:
205  return class_name
206  for n in range(0, len(class_name)):
207  if class_name[n].isupper() and n != 0:
208  class_name = str(class_name[0:n]) + "".join(["_", class_name[n].lower()]) + str(class_name[n+1:])
209  elif class_name[n].isupper() and n == 0:
210  class_name = str(class_name[0:n]) + "".join([class_name[n].lower()]) + str(class_name[n+1:])
211  return class_name
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def querying.connection.close_session (   self)

Definition at line 120 of file querying.py.

References cmsRelvalreport.exit.

120 
121  def close_session(self):
122  try:
123  self.session.close()
124  return True
125  except Exception as e:
126  exit(e)
def querying.connection.commit (   self)

Definition at line 305 of file querying.py.

References querying.connection.connection_data, reco.print(), and querying.connection.session.

Referenced by querying.connection.write_and_commit().

305 
306  def commit(self):
307  if self.connection_data["host"] == "sqlite":
308  if self.session != None:
309  self.session.commit()
310  else:
311  print("Writing to non-sqlite databases currently not supported.")
reco::print
std::string print(const Track &, edm::Verbosity=edm::Concise)
Track print utility.
Definition: print.cc:10
def querying.connection.deepcopy_model (   self,
  model 
)

Definition at line 115 of file querying.py.

References cmsPerfStripChart.dict.

115 
116  def deepcopy_model(self, model):
117  new_dict = dict(model.__dict__)
118  new_dict["__table__"] = deepcopy(model.__dict__["__table__"])
119  return type(model.__class__.__name__, (), new_dict)
def querying.connection.global_tag (   self,
  pkargs 
)

Definition at line 230 of file querying.py.

230 
231  def global_tag(self, **pkargs):
232  return self.factory.object("globaltag", **pkargs)
def querying.connection.global_tag_map (   self,
  pkargs 
)

Definition at line 233 of file querying.py.

233 
234  def global_tag_map(self, **pkargs):
235  return self.factory.object("globaltagmap", **pkargs)
def querying.connection.global_tag_map_request (   self,
  pkargs 
)

Definition at line 236 of file querying.py.

236 
237  def global_tag_map_request(self, **pkargs):
238  return self.factory.object("globaltagmaprequest", **pkargs)
querying.connection.global_tag_map_request
def global_tag_map_request
Definition: querying.py:236
def querying.connection.iov (   self,
  pkargs 
)

Definition at line 242 of file querying.py.

242 
243  def iov(self, **pkargs):
244  return self.factory.object("iov", **pkargs)
def querying.connection.model (   self,
  model_name 
)

Definition at line 213 of file querying.py.

References querying.connection.models.

Referenced by querying.connection.search_everything(), Vispa.Gui.Header.Header.setText(), and Vispa.Gui.SimpleDraggableTreeWidget.SimpleDraggableTreeWidget.startDrag().

213 
214  def model(self, model_name):
215  if model_name.__class__ == sqlalchemy.ext.declarative.api.DeclarativeMeta:
216  model_name = model_name.__name__
217  model_name = model_name.replace("_", "")
218  return self.models[model_name]
def querying.connection.object (   self,
  model,
  pk_to_value 
)

Definition at line 222 of file querying.py.

References querying.connection.session.

Referenced by Vispa.Views.LineDecayView.LineDecayContainer.createObject(), Vispa.Views.LineDecayView.LineDecayContainer.delete(), Vispa.Views.LineDecayView.DecayLine.delete(), Vispa.Views.LineDecayView.LineDecayContainer.mouseReleaseEvent(), and Vispa.Views.BoxDecayView.BoxDecayContainer.toggleCollapse().

222 
223  def object(self, model, pk_to_value):
224  if self.session == None:
225  return None
226  model_data = self.session.query(model)
227  for pk in pk_to_value:
228  model_data = model_data.filter(model.__dict__[pk] == pk_to_value[pk])
229  return model_data.first()
def querying.connection.payload (   self,
  pkargs 
)

Definition at line 245 of file querying.py.

245 
246  def payload(self, **pkargs):
247  return self.factory.object("payload", **pkargs)
def querying.connection.record (   self,
  pkargs 
)

Definition at line 248 of file querying.py.

248 
249  def record(self, **pkargs):
250  return self.factory.object("payload", **pkargs)
def querying.connection.search_everything (   self,
  string,
  amount = 10 
)

Definition at line 256 of file querying.py.

References querying.connection._oracle_match_format(), alcazmumu_cfi.filter, MessageLogger_cff.limit, and querying.connection.model().

256 
257  def search_everything(self, string, amount=10):
258  string = self._oracle_match_format(string)
259 
260  gt = self.model("globaltag")
261  global_tags = self.session.query(gt).filter(or_(
262  gt.name.ilike(string),
263  gt.description.ilike(string),
264  gt.release.ilike(string)
265  )).limit(amount)
266  tag = self.model("tag")
267  tags = self.session.query(tag).filter(or_(
268  tag.name.ilike(string),
269  tag.object_type.ilike(string),
270  tag.description.ilike(string))
271  ).limit(amount)
272  iov = self.model("iov")
273  iovs = self.session.query(iov).filter(or_(
274  iov.tag_name.ilike(string),
275  iov.since.ilike(string),
276  iov.payload_hash.ilike(string),
277  iov.insertion_time.ilike(string)
278  )).limit(amount)
279  payload = self.model("payload")
280  payloads = self.session.query(payload).filter(or_(
281  payload.hash.ilike(string),
282  payload.object_type.ilike(string),
283  payload.insertion_time.ilike(string)
284  )).limit(amount)
285 
286  return json_data_node.make({
287  "global_tags" : global_tags.all(),
288  "tags" : tags.all(),
289  "iovs" : iovs.all(),
290  "payloads" : payloads.all()
291  })
MessageLogger_cff.limit
tuple limit
Definition: MessageLogger_cff.py:11
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def querying.connection.setup (   self)

Definition at line 73 of file querying.py.

73 
74  def setup(self):
75 
76  self.db_name = self.connection_data["db_alias"]
77 
78  if self.connection_data["host"] != "sqlite":
79  if self.connection_data["host"] != "frontier":
80  # if not frontier, we have to authenticate
81  user = self.secrets["login"]
82  pwd = self.secrets["password"]
83  self.engine = create_engine(self.build_oracle_url(user, pwd, self.db_name))
84  else:
85  # if frontier, no need to authenticate
86  self.engine = create_engine(self.build_frontier_url(self.db_name, self.schema))
87  else:
88  # if host is sqlite, making the url is easy - no authentication
89  self.engine = create_engine("sqlite:///%s" % self.db_name)
90 
91  self.sessionmaker = sessionmaker(bind=self.engine)
92  self.session = self.sessionmaker()
93  self.factory = factory(self)
94 
95  # assign correct schema for database name to each model
96  tmp_models_dict = {}
97  for key in self.models:
98  try:
99  if self.models[key].__class__ == sqlalchemy.ext.declarative.api.DeclarativeMeta\
100  and str(self.models[key].__name__) != "Base":
101 
102  self.models[key].__table__.schema = self.schema
103 
104  self.models[key].session = self.session
105  self.models[key].authentication = self.netrc_authenticators
106  self.models[key].secrets = self.secrets
107  tmp_models_dict[key.lower()] = self.models[key]
108  tmp_models_dict[key.lower()].empty = False
109  except AttributeError:
110  continue
111 
112  self.models = tmp_models_dict
113 
114  return self
def querying.connection.tag (   self,
  pkargs 
)

Definition at line 239 of file querying.py.

Referenced by Inspector.Inspector.SetTag().

239 
240  def tag(self, **pkargs):
241  return self.factory.object("tag", **pkargs)
def querying.connection.write (   self,
  object 
)

Definition at line 294 of file querying.py.

References querying.connection.connection_data, reco.print(), querying.connection.schema, and querying.connection.session.

Referenced by pkg.AbstractPkg.generate(), and querying.connection.write_and_commit().

294 
295  def write(self, object):
296  if self.connection_data["host"] == "sqlite":
297  if self.session != None:
298  class_of_object = object.__class__
299  new_object = class_of_object(object.as_dicts(), convert_timestamps=False)
300  new_object.__table__.schema = self.schema
301  self.session.add(new_object)
302  return new_object
303  else:
304  print("Writing to non-sqlite databases currently not supported.")
reco::print
std::string print(const Track &, edm::Verbosity=edm::Concise)
Track print utility.
Definition: print.cc:10
def querying.connection.write_and_commit (   self,
  object 
)

Definition at line 312 of file querying.py.

References cond::persistency::CondDBTransaction.commit(), ora::Transaction.commit(), cond::persistency::ITransaction.commit(), ora::ScopedTransaction.commit(), cond::persistency::Transaction.commit(), cond::persistency::TransactionScope.commit(), querying.connection.commit(), ora::PVectorWriter.write(), pos::PixelConfig.write(), AlignmentCorrelationsIO.write(), ora::IBlobStreamingService.write(), ora::IRelationalWriter.write(), AlignmentCorrelationsIORoot.write(), SurveyOutput.write(), ora::QueryableVectorWriter.write(), ora::CArrayWriter.write(), ora::OraPtrWriter.write(), ora::UniqueRefWriter.write(), AlignmentParametersIO.write(), AlignmentUserVariablesIO.write(), ora::STLContainerWriter.write(), popcon::PopConAnalyzer< S >.write(), ora::ObjectWriter.write(), popcon::PopCon.write(), ora::PrimitiveWriter.write(), MuonAlignmentOutputXML.write(), ora::InlineCArrayWriter.write(), ora::OraReferenceWriter.write(), ora::BlobWriter.write(), ora::DependentClassWriter.write(), CSCPairResidualsConstraint.write(), XMLDOMBlock.write(), ora::NamedRefWriter.write(), ora::ITokenWriter.write(), XMLDocument.write, MuonResidualsTwoBin.write(), XMLProcessor.write(), MuonResidualsFitter.write(), betterConfigParser.BetterConfigParser.write(), and querying.connection.write().

312 
313  def write_and_commit(self, object):
314  # should be changed to deal with errors - add them to exception handling if they appear
315  self.write(object)
316  self.commit()
317 
318 
# contains methods for creating objects

Member Data Documentation

querying.connection.base = None
static

Definition at line 21 of file querying.py.

querying.connection.connection = None
static

Definition at line 18 of file querying.py.

Referenced by testing_classes.querying_tests.test_check_connection(), and testing_classes.data_formats_tests.test_dicts_to_orm_objects().

querying.connection.connection_data = None
static

Definition at line 20 of file querying.py.

Referenced by querying.connection.__init__(), querying.connection.commit(), testing_classes.script_tests.test_script(), testing_classes.script_tests.test_script_with_decorator(), and querying.connection.write().

querying.connection.db_name

Definition at line 75 of file querying.py.

Referenced by querying.connection.__repr__(), compare_using_files_v2.RootFileComparison.compare(), ValidationMatrix_v2.ReleaseComparison.compare(), compare_using_files_v2.RootFileComparison.was_compared(), and ValidationMatrix_v2.ReleaseComparison.was_compared().

querying.connection.engine = None
static

Definition at line 17 of file querying.py.

Referenced by conddblib.Connection.init().

querying.connection.factory

Definition at line 92 of file querying.py.

querying.connection.models

Definition at line 69 of file querying.py.

Referenced by querying.connection.model().

querying.connection.netrc_authenticators = None
static

Definition at line 22 of file querying.py.

int querying.connection.row_limit = 1000
static

Definition at line 16 of file querying.py.

querying.connection.schema

Definition at line 58 of file querying.py.

Referenced by querying.connection.write().

querying.connection.secrets = None
static

Definition at line 23 of file querying.py.

querying.connection.session = None
static

Definition at line 19 of file querying.py.

Referenced by querying.connection.commit(), querying.connection.object(), and querying.connection.write().

querying.connection.sessionmaker

Definition at line 90 of file querying.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5