CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
electronDbsDiscovery.py
Go to the documentation of this file.
1 
2 #===================================================================
3 # This python module is querying https://cmsweb.cern.ch/dbs_discovery/
4 # so to get the list of input files. One must call :
5 # search(), to get the list of primary files
6 # search2(), to get the list of eventual secondary files
7 #
8 # The selection of files is configured thanks to shell
9 # environment variables:
10 #
11 # DBS_RELEASE, for example CMSSW_2_2_0_pre1
12 # DBS_SAMPLE, for example RelValSingleElectronPt35
13 # DBS_RUN, for example Any
14 # DBS_COND , for example MC_31X_V2-v1
15 # DBS_TIER , for example RECO
16 # DBS_TIER_SECONDARY, for eventual secondary files
17 #
18 # DBS_STRATEGY:
19 # search: use dbs search
20 # castor: use rfdir
21 # lsf: use dbs lsf
22 # local: look within electronDbsDiscovery.txt
23 #
24 # In DBS_COND, DBS_TIER and DBS_TIER_SECONDARY,
25 # one can use wildcard *.
26 #===================================================================
27 
28 #import httplib, urllib, urllib2, types, string, os, sys
29 import os, sys
30 
31 if not os.environ.has_key('DBS_RELEASE'):
32  os.environ['DBS_RELEASE'] = "Any"
33 if not os.environ.has_key('DBS_SAMPLE'):
34  os.environ['DBS_SAMPLE'] = "Any"
35 if not os.environ.has_key('DBS_RUN'):
36  os.environ['DBS_RUN'] = "Any"
37 if not os.environ.has_key('DBS_TIER_SECONDARY'):
38  os.environ['DBS_TIER_SECONDARY'] = ""
39 if not os.environ.has_key('DBS_STRATEGY'):
40  os.environ['DBS_STRATEGY'] = "search"
41 
42 def common_search(dbs_tier):
43 
44  if os.environ['DBS_STRATEGY'] == "local":
45 
46  result = []
47  for line in open('electronDbsDiscovery.txt').readlines():
48  line = os.path.expandvars(line.strip())
49  if line == "": continue
50  if os.environ['DBS_SAMPLE'] != "Any" and line.find(os.environ['DBS_SAMPLE'])== -1: continue
51  if line.find(os.environ['DBS_COND'])== -1: continue
52  if line.find(dbs_tier)== -1: continue
53  result.append('file:'+line)
54 
55  elif os.environ['DBS_STRATEGY'] == "castor":
56 
57  castor_dir = '/castor/cern.ch/cms/store/relval/'+os.environ['DBS_RELEASE']+'/'+os.environ['DBS_SAMPLE']+'/'+os.environ['DBS_TIER']+'/'+os.environ['DBS_COND']+'/'
58  if __name__ == "__main__":
59  print 'castor dir:',castor_dir
60  result = []
61  data = os.popen('rfdir '+castor_dir)
62  subdirs = data.readlines()
63  data.close()
64  datalines = []
65  for line in subdirs:
66  line = line.rstrip()
67  subdir = line.split()[8]
68  data = os.popen('rfdir '+castor_dir+'/'+subdir)
69  datalines = data.readlines()
70  for line in datalines:
71  line = line.rstrip()
72  file = line.split()[8]
73  if file != "":
74  result.append('/store/relval/'+os.environ['DBS_RELEASE']+'/'+os.environ['DBS_SAMPLE']+'/'+os.environ['DBS_TIER']+'/'+os.environ['DBS_COND']+'/'+subdir+'/'+file)
75  data.close()
76 
77  elif os.environ['DBS_STRATEGY'] == "lsf":
78 
79  dbs_path = '/'+os.environ['DBS_SAMPLE']+'/'+os.environ['DBS_RELEASE']+'-'+os.environ['DBS_COND']+'/'+os.environ['DBS_TIER']+'"'
80  if __name__ == "__main__":
81  print 'dbs path:',dbs_path
82  data = os.popen('dbs lsf --path="'+dbs_path+'"')
83  datalines = data.readlines()
84  data.close()
85  result = []
86  for line in datalines:
87  line = line.rstrip()
88  if line != "" and line[0] =="/":
89  result.append(line)
90 
91  else:
92 
93  input = "find file"
94  separator = " where "
95  if os.environ['DBS_RELEASE'] != "Any":
96  input = input + separator + "release = " + os.environ['DBS_RELEASE']
97  separator = " and "
98  if os.environ['DBS_SAMPLE'] != "Any":
99  input = input + separator + "primds = " + os.environ['DBS_SAMPLE']
100  separator = " and "
101  if os.environ['DBS_RUN'] != "Any":
102  input = input + separator + "run = " + os.environ['DBS_RUN']
103  separator = " and "
104  input = input + separator + "dataset like *" + os.environ['DBS_COND'] + "*" + dbs_tier + "*"
105 
106  #url = "https://cmsweb.cern.ch:443/dbs_discovery/aSearch"
107  #final_input = urllib.quote(input) ;
108  #
109  #agent = "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"
110  #ctypes = "text/plain"
111  #headers = { 'User-Agent':agent, 'Accept':ctypes}
112  #params = {'dbsInst':'cms_dbs_prod_global',
113  # 'html':0,'caseSensitive':'on','_idx':0,'pagerStep':-1,
114  # 'userInput':final_input,
115  # 'xml':0,'details':0,'cff':0,'method':'dbsapi'}
116  #data = urllib.urlencode(params,doseq=True)
117  #req = urllib2.Request(url, data, headers)
118  #data = ""
119  #
120  #try:
121  # response = urllib2.urlopen(req)
122  # data = response.read()
123  #except urllib2.HTTPError, e:
124  # if e.code==201:
125  # print e.headers
126  # print e.msg
127  # pass
128  # else:
129  # raise e
130 
131  data = os.popen('dbs search --url="http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" --query "'+input+'"')
132  datalines = data.readlines()
133  data.close()
134  result = []
135  for line in datalines:
136  line = line.rstrip()
137  if line != "" and line[0] =="/":
138  result.append(line)
139 
140  return result
141 
142 def search():
143  return common_search(os.environ['DBS_TIER'])
144 
145 def search2():
146  return common_search(os.environ['DBS_TIER_SECONDARY'])
147 
148 
149 
150