CMS 3D CMS Logo
Functions | Variables
electronDataDiscovery Namespace Reference

Functions

def common_search (dd_tier)
 
def getCMSdata (data, dbs="prod/global")
 
def search ()
 
def search2 ()
 

Variables

 dd_cond_re
 
 dd_release_re
 
 dd_run_re
 
 dd_sample_re
 
 files
 

Function Documentation

◆ common_search()

def electronDataDiscovery.common_search (   dd_tier)

Definition at line 60 of file electronDataDiscovery.py.

References das_client.get_data(), print(), isotrackApplyRegressor.range, python.rootplot.root2matplotlib.replace(), and str.

Referenced by search(), and search2().

60 def common_search(dd_tier):
61  dd_tier_re = re.compile(dd_tier.replace('*', '.*'));
62 
63  if os.environ['DD_SOURCE'] == "das":
64 
65  query = "dataset instance=cms_dbs_prod_global"
66  if os.environ['DD_RELEASE'] != "":
67  query = query + " release=" + os.environ['DD_RELEASE']
68  if os.environ['DD_SAMPLE'] != "":
69  query = query + " primary_dataset=" + os.environ['DD_SAMPLE']
70  if dd_tier != "":
71  query = query + " tier=" + dd_tier
72  if os.environ['DD_COND'] != "":
73  query = query + " dataset=*" + os.environ['DD_COND'] + "*"
74  if os.environ['DD_RUN'] != "":
75  query = query + " run=" + os.environ['DD_RUN']
76  # query = query + " | unique" # too long ??
77 
78  # data = os.popen('das_client.py --limit=0 --query "'+query+'"')
79  # datalines = data.readlines()
80  # data.close()
81  # datasets = []
82  # for line in datalines:
83  # line = line.rstrip()
84  # if line != "" and line[0] =="/":
85  # datasets.append(line)
86  # dataset = datasets[0]
87 
88  data = das_client.json.loads(das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, 0))
89 
90  if data['nresults'] == 0:
91  print('[electronDataDiscovery.py] No DAS dataset for query:', query)
92  return []
93  while data['nresults'] > 1:
94  if data['data'][0]['dataset'][0]['name'] == data['data'][1]['dataset'][0]['name']:
95  data['data'].pop(0)
96  data['nresults'] -= 1
97  else:
98  print('[electronDataDiscovery.py] Several DAS datasets for query:', query)
99  for i in range(data['nresults']):
100  print(
101  '[electronDataDiscovery.py] dataset[' + str(i) + ']: ' + data['data'][i]['dataset'][0]['name'])
102  return []
103 
104  dataset = data['data'][0]['dataset'][0]['name']
105 
106  query = "file instance=cms_dbs_prod_global dataset=" + dataset
107 
108  # data = os.popen('das_client.py --limit=0 --query "'+query+'"')
109  # datalines = data.readlines()
110  # data.close()
111  # result = []
112  # for line in datalines:
113  # line = line.rstrip()
114  # if line != "" and line[0] =="/":
115  # result.append(line)
116 
117  data = das_client.json.loads(das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, 0))
118 
119  if data['nresults'] == 0:
120  print('[electronDataDiscovery.py] No DAS file in dataset:', dataset)
121  return []
122  else:
123  print('there is %d results' % nresults)
124 
125  result = []
126  for i in range(0, data['nresults']):
127  result.append(str(data['data'][i]['file'][0]['name']))
128 
129  elif os.environ['DD_SOURCE'] == "dbs":
130 
131  input = "find file"
132  separator = " where "
133  if os.environ['DD_RELEASE'] != "":
134  input = input + separator + "release = " + os.environ['DD_RELEASE']
135  separator = " and "
136  if os.environ['DD_SAMPLE'] != "":
137  input = input + separator + "primds = " + os.environ['DD_SAMPLE']
138  separator = " and "
139  if os.environ['DD_RUN'] != "":
140  input = input + separator + "run = " + os.environ['DD_RUN']
141  separator = " and "
142  input = input + separator + "dataset like *" + os.environ['DD_COND'] + "*" + dd_tier + "*"
143 
144  data = os.popen(
145  'dbs search --url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" --query "' + input + '"')
146  datalines = data.readlines()
147  data.close()
148  result = []
149  for line in datalines:
150  line = line.rstrip()
151  if line != "" and line[0] == "/":
152  result.append(line)
153 
154  elif os.environ['DD_SOURCE'] == "http":
155 
156  input = "find file"
157  separator = " where "
158  if os.environ['DD_RELEASE'] != "":
159  input = input + separator + "release = " + os.environ['DD_RELEASE']
160  separator = " and "
161  if os.environ['DD_SAMPLE'] != "":
162  input = input + separator + "primds = " + os.environ['DD_SAMPLE']
163  separator = " and "
164  if os.environ['DD_RUN'] != "":
165  input = input + separator + "run = " + os.environ['DD_RUN']
166  separator = " and "
167  input = input + separator + "dataset like *" + os.environ['DD_COND'] + "*" + dd_tier + "*"
168 
169  url = "https://cmsweb.cern.ch:443/dbs_discovery/aSearch"
170  final_input = urllib.quote(input);
171 
172  agent = "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"
173  ctypes = "text/plain"
174  headers = {'User-Agent': agent, 'Accept': ctypes}
175  params = {'dbsInst': 'cms_dbs_prod_global',
176  'html': 0, 'caseSensitive': 'on', '_idx': 0, 'pagerStep': -1,
177  'userInput': final_input,
178  'xml': 0, 'details': 0, 'cff': 0, 'method': 'dbsapi'}
179  data = urllib.urlencode(params, doseq=True)
180  req = urllib2.Request(url, data, headers)
181  data = ""
182 
183  try:
184  response = urllib2.urlopen(req)
185  data = response.read()
186  except urllib2.HTTPError as e:
187  if e.code == 201:
188  print(e.headers)
189  print(e.msg)
190  pass
191  else:
192  raise e
193 
194  datalines = data.readlines()
195  data.close()
196  result = []
197  for line in datalines:
198  line = line.rstrip()
199  if line != "" and line[0] == "/":
200  result.append(line)
201 
202  elif os.environ['DD_SOURCE'] == "lsf":
203 
204  dbs_path = '/' + os.environ['DD_SAMPLE'] + '/' + os.environ['DD_RELEASE'] + '-' + os.environ['DD_COND'] + '/' + \
205  os.environ['DD_TIER'] + '"'
206  if __name__ == "__main__":
207  print('dbs path:', dbs_path)
208  data = os.popen('dbs lsf --path="' + dbs_path + '"')
209  datalines = data.readlines()
210  data.close()
211  result = []
212  for line in datalines:
213  line = line.rstrip()
214  if line != "" and line[0] == "/":
215  result.append(line)
216 
217  elif os.environ['DD_SOURCE'].startswith('/castor/cern.ch/cms/'): # assumed to be a castor dir
218 
219  castor_dir = os.environ['DD_SOURCE'].replace('/castor/cern.ch/cms/', '/', 1)
220  result = []
221  data = os.popen('rfdir /castor/cern.ch/cms' + castor_dir)
222  subdirs = data.readlines()
223  data.close()
224  datalines = []
225  for line in subdirs:
226  line = line.rstrip()
227  subdir = line.split()[8]
228  data = os.popen('rfdir /castor/cern.ch/cms' + castor_dir + '/' + subdir)
229  datalines = data.readlines()
230  for line in datalines:
231  line = line.rstrip()
232  file = line.split()[8]
233  if file != "":
234  result.append(castor_dir + '/' + subdir + '/' + file)
235  data.close()
236 
237  elif os.environ['DD_SOURCE'].startswith('/eos/cms/'): # assumed to be an eos dir
238 
239  data = os.popen('eos find -f ' + os.environ['DD_SOURCE'])
240  lines = data.readlines()
241  data.close()
242  result = []
243  for line in lines:
244  line = line.strip().replace('/eos/cms/', '/', 1)
245  if line == "": continue
246  if dd_sample_re.search(line) == None: continue
247  if dd_cond_re.search(line) == None: continue
248  if dd_tier_re.search(line) == None: continue
249  if dd_run_re.search(line) == None: continue
250  result.append(line)
251 
252  else: # os.environ['DD_SOURCE'] is assumed to be a file name
253 
254  result = []
255  for line in open(os.environ['DD_SOURCE']).readlines():
256  line = os.path.expandvars(line.strip())
257  if line == "": continue
258  if dd_sample_re.search(line) == None: continue
259  if dd_cond_re.search(line) == None: continue
260  if dd_tier_re.search(line) == None: continue
261  if dd_run_re.search(line) == None: continue
262  result.append(line)
263 
264  if len(result) == 0:
265  diag = '[electronDataDiscovery.py] No more files after filtering with :'
266  if os.environ['DD_SAMPLE'] != '': diag += ' ' + os.environ['DD_SAMPLE']
267  if os.environ['DD_COND'] != '': diag += ' ' + os.environ['DD_COND']
268  if dd_tier != '': diag += ' ' + dd_tier
269  if os.environ['DD_RUN'] != '': diag += ' ' + os.environ['DD_RUN']
270  print(diag)
271 
272  return result
273 
274 
das_client.get_data
def get_data(host, query, idx, limit, debug, threshold=300, ckey=None, cert=None, capath=None, qcache=0, das_headers=True)
Definition: das_client.py:276
python.rootplot.root2matplotlib.replace
def replace(string, replacements)
Definition: root2matplotlib.py:444
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
str
#define str(s)
Definition: TestProcessor.cc:56

◆ getCMSdata()

def electronDataDiscovery.getCMSdata (   data,
  dbs = "prod/global" 
)

Definition at line 284 of file electronDataDiscovery.py.

References SiPixelLorentzAngle_cfi.read, and python.rootplot.root2matplotlib.replace().

284 def getCMSdata(data, dbs="prod/global"):
285  # Read DAS database.
286  cmd = 'dasgoclient --query="file dataset=DATA instance=DBS" | sort'
287  cmd2 = cmd.replace('DATA', data).replace('DBS', dbs)
288  files = os.popen(cmd2).read()
289  # Create python list containing file names.
290  flist = files.split('\n')
291  del flist[-1]
292  return flist
293 
python.rootplot.root2matplotlib.replace
def replace(string, replacements)
Definition: root2matplotlib.py:444
electronDataDiscovery.getCMSdata
def getCMSdata(data, dbs="prod/global")
Definition: electronDataDiscovery.py:284
SiPixelLorentzAngle_cfi.read
read
Definition: SiPixelLorentzAngle_cfi.py:7

◆ search()

def electronDataDiscovery.search ( )

Definition at line 275 of file electronDataDiscovery.py.

References common_search(), and print().

275 def search():
276  print('search in %s' % 'DD_TIER')
277  return common_search(os.environ['DD_TIER'])
278 
279 
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

◆ search2()

def electronDataDiscovery.search2 ( )

Definition at line 280 of file electronDataDiscovery.py.

References common_search().

Referenced by DQMPFCandidateAnalyzer.dqmBeginRun(), and METAnalyzer.dqmBeginRun().

280 def search2():
281  return common_search(os.environ['DD_TIER_SECONDARY'])
282 
283 

Variable Documentation

◆ dd_cond_re

electronDataDiscovery.dd_cond_re

Definition at line 56 of file electronDataDiscovery.py.

◆ dd_release_re

electronDataDiscovery.dd_release_re

Definition at line 54 of file electronDataDiscovery.py.

◆ dd_run_re

electronDataDiscovery.dd_run_re

Definition at line 57 of file electronDataDiscovery.py.

◆ dd_sample_re

electronDataDiscovery.dd_sample_re

Definition at line 55 of file electronDataDiscovery.py.

◆ files

electronDataDiscovery.files

Definition at line 11 of file electronDataDiscovery.py.

Generated for CMSSW Reference Manual by doxygen 1.8.14