CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
cmsHarvester Namespace Reference

Classes

class  CMSHarvester
 CMSHarvester class. More...
 
class  CMSHarvesterHelpFormatter
 Helper class: CMSHarvesterHelpFormatter. More...
 
class  DBSXMLHandler
 Helper class: DBSXMLHandler. More...
 
class  Error
 Helper class: Error exception. More...
 
class  Usage
 Helper class: Usage exception. More...
 

Functions

def build_dataset_ignore_list
 
def build_dataset_list
 def dbs_check_dataset_num_events(self, dataset_name): """Figure out the number of events in each run of this dataset. More...
 
def build_dataset_use_list
 
def build_datasets_information
 
def build_runs_ignore_list
 
def build_runs_list
 
def build_runs_use_list
 
def check_cmssw
 
def check_dataset_list
 
def check_dbs
 
def check_globaltag
 
def check_globaltag_contains_ref_hist_key
 
def check_globaltag_exists
 
def check_input_status
 
def check_ref_hist_mappings
 
def check_ref_hist_tag
 
def create_and_check_castor_dir
 
def create_and_check_castor_dirs
 
def create_castor_path_name_common
 
def create_castor_path_name_special
 
def create_config_file_name
 
def create_crab_config
 
def create_es_prefer_snippet
 
def create_harvesting_config
 
def create_harvesting_config_file_name
 
def create_harvesting_output_file_name
 
def create_me_extraction_config
 
More...
 
def create_me_summary_config_file_name
 
def create_me_summary_output_file_name
 
def create_multicrab_block_name
 
def create_multicrab_config
 
def create_output_file_name
 
def dbs_check_dataset_spread
 def dbs_resolve_dataset_number_of_sites(self, dataset_name): """Ask DBS across how many sites this dataset has been spread out. More...
 
def dbs_resolve_cmssw_version
 
def dbs_resolve_dataset_name
 
def dbs_resolve_datatype
 
def dbs_resolve_globaltag
 
def dbs_resolve_number_of_events
 
def dbs_resolve_runs
 def dbs_resolve_dataset_number_of_events(self, dataset_name): """Ask DBS across how many events this dataset has been spread out. More...
 
def escape_dataset_name
 
def load_ref_hist_mappings
 
def option_handler_caf_access
 
def option_handler_castor_dir
 def option_handler_dataset_name(self, option, opt_str, value, parser): """Specify the name(s) of the dataset(s) to be processed. More...
 
def option_handler_crab_submission
 
def option_handler_list_types
 
def option_handler_no_t1access
 
def option_handler_preferred_site
 
def option_handler_saveByLumiSection
 
def option_handler_sites
 
def parse_cmd_line_options
 
def pick_a_site
 
def process_dataset_ignore_list
 
def process_runs_use_and_ignore_lists
 
def ref_hist_mappings_needed
 
def run
 
def setup_dbs
 

Now we try to do a very simple DBS search.

More...
 
def setup_harvesting_info
 
def show_exit_message
 
def singlify_datasets
 
def write_crab_config
 def create_harvesting_config(self, dataset_name): """Create the Python harvesting configuration for a given job. More...
 
def write_harvesting_config
 
def write_me_extraction_config
 
def write_multicrab_config
 

Variables

string __author__ = "Jeroen Hegeman (jeroen.hegeman@cern.ch),"
 
string __version__ = "3.8.2p1"
 File : cmsHarvest.py Authors : Jeroen Hegeman (jeroe.nosp@m.n.he.nosp@m.geman.nosp@m.@cer.nosp@m.n.ch) Niklas Pietsch (nikla.nosp@m.s.pi.nosp@m.etsch.nosp@m.@des.nosp@m.y.de) Franseco Costanza (franc.nosp@m.esco.nosp@m..cost.nosp@m.anza.nosp@m.@desy.nosp@m..de) Last change: 20100308. More...
 
string action = "callback"
 
list all_file_names = files_info[run_number]
 
list all_t1
 
 caf_access
 
 callback = self.option_handler_input_Jsonrunfile,
 
 castor_base_dir
 
list castor_dir = self.datasets_information[dataset_name]
 

CRAB

More...
 
tuple castor_path_common = self.create_castor_path_name_common(dataset_name)
 

DEBUG DEBUG DEBUG

This is probably only useful to make sure we don't muck

things up, right?

Figure out across how many sites this sample has been spread.

More...
 
tuple castor_paths
 
 castor_prefix = self.castor_prefix
 
string cmd = "rfstat %s"
 self.logger.debug("Path is now `%s'" % \ path) More...
 
list cmssw_version = self.datasets_information[dataset_name]
 
list complete_sites
 site_names_ref = set(files_info[run_number].values()[0][1]) for site_names_tmp in files_info[run_number].values()[1:]: if set(site_names_tmp[1]) != site_names_ref: mirrored = False break More...
 
tuple config_builder = ConfigBuilder(config_options, with_input=True)
 
 config_contents = config_builder.pythonCfgCode
 

In case this file is the second step (the real harvesting

step) of the two-step harvesting we have to tell it to use

our local files.

More...
 
tuple config_file_name = self.create_me_summary_config_file_name(dataset_name)
 

Only add the alarming piece to the file name if this is

a spread-out dataset.

More...
 
list connect_name = self.frontier_connection_name["globaltag"]
 
dictionary country_codes
 
string crab_config = "\n"
 

CRAB

More...
 
 crab_submission
 
list customisations = [""]
 
tuple dataset_name_escaped = self.escape_dataset_name(dataset_name)
 
tuple dataset_names = self.datasets_to_use.keys()
 
 dataset_names_after_checks = dataset_names_after_checks_tmp
 
tuple dataset_names_after_checks_tmp = copy.deepcopy(dataset_names_after_checks)
 
 datasets_information
 
 datasets_to_ignore
 
 datasets_to_use
 
list datatype = self.datasets_information[dataset_name]
 
 dbs_api
 
tuple empty_runs = dict(tmp)
 
tuple es_prefer_snippet = self.create_es_prefer_snippet(dataset_name)
 
int exit_code = 1
 
list file_name = handler.results["file.name"]
 
list files_at_site
 
dictionary files_info = {}
 
list files_without_sites
 
list globaltag = self.datasets_information[dataset_name]
 
 harvesting_info
 
 harvesting_mode
 
 harvesting_type
 
string help = "Jsonfile containing dictionary of run/lumisections pairs. "
 
string index = "site_%02d"
 
 Jsonfilename
 
 Jsonlumi
 
int loop = 0
 

CMSSW

More...
 
string marker = "\n"
 
list marker_lines = []
 
string metavar = "JSONRUNFILE"
 
 mirrored = None
 
string msg = "Could not create directory `%s'"
 class Handler(xml.sax.handler.ContentHandler): def startElement(self, name, attrs): if name == "result": site_name = str(attrs["STORAGEELEMENT_SENAME"])

TODO TODO TODO

Ugly hack to get around cases like this:

$ dbs search –query="find dataset, site, file.count where dataset=/RelValQCD_Pt_3000_3500/CMSSW_3_3_0_pre1-STARTUP31X_V4-v1/GEN-SIM-RECO"

Using DBS instance at: http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet

Processing ...

More...
 
tuple multicrab_block_name
 
string multicrab_config = "\n"
 
list multicrab_config_lines = []
 
tuple nevents = int(handler.results["file.numevents"][index])
 
 non_t1access
 
 nr_max_sites
 
dictionary num_events_catalog = {}
 
tuple num_events_dataset = sum(tmp)
 
tuple num_sites
 
               if self.datasets_information[dataset_name]["num_events"][run_number] != 0:
                   pdb.set_trace()

DEBUG DEBUG DEBUG end More...

 
int number_max_sites = self.nr_max_sites+1
 
 option_parser
 
 output_file_name = self.\
 
tuple path = os.path.join(path, piece)
 else:

Piece not in the list, fine.

More...
 
tuple permissions = extract_permissions(output)
 
list permissions_new = []
 
string permissions_target = "775"
 
 preferred_site
 
 ref_hist_mappings_file_name
 
tuple run_number = int(handler.results["run.number"][index])
 
list runs = self.datasets_to_use[dataset_name]
 
 runs_to_ignore
 
 runs_to_use
 
 saveByLumiSection
 
tuple se_name = choice(t1_sites)
 
string sep = "#"
 
 site_name = None
 
tuple site_names = list(set([j for i in files_info[run_number].values() for j in i[1]]))
 
list sites = [self.preferred_site]
 
list sites_forbidden = []
 
list sites_with_complete_copies = []
 
 skip_this_path_piece = True
 self.logger.debug("Checking CASTOR path piece `%s'" % \ piece) More...
 
list t1_sites = []
 
list tmp
 

TODO TODO TODO

Need to think about where this should go, but

somewhere we have to move over the fact that we want

to process all runs for each dataset that we're

considering.

More...
 
tuple traceback_string = traceback.format_exc()
 
string twiki_url = "https://twiki.cern.ch/twiki/bin/view/CMS/CmsHarvester"
 
string type = "string"
 
tuple use_es_prefer = (self.harvesting_type == "RelVal")
 
 use_refs = use_es_preferor\
 
 UserName = output
 
 workflow_name = dataset_name
 

Function Documentation

def cmsHarvester.build_dataset_ignore_list (   self) 
Build a list of datasets to ignore.

NOTE: We should always have a list of datasets to process, but
it may be that we don't have a list of datasets to ignore.

Definition at line 3444 of file cmsHarvester.py.

3444 
3445  def build_dataset_ignore_list(self):
3446  """Build a list of datasets to ignore.
3447 
3448  NOTE: We should always have a list of datasets to process, but
3449  it may be that we don't have a list of datasets to ignore.
3450 
3451  """
3452 
3453  self.logger.info("Building list of datasets to ignore...")
3454 
3455  input_method = self.input_method["datasets"]["ignore"]
3456  input_name = self.input_name["datasets"]["ignore"]
3457  dataset_names = self.build_dataset_list(input_method,
3458  input_name)
3459  self.datasets_to_ignore = dict(zip(dataset_names,
3460  [None] * len(dataset_names)))
3461 
3462  self.logger.info(" found %d dataset(s) to ignore:" % \
3463  len(dataset_names))
3464  for dataset in dataset_names:
3465  self.logger.info(" `%s'" % dataset)
3466 
3467  # End of build_dataset_ignore_list.
archive.zip
tuple zip
Definition: archive.py:476
cmsHarvester.build_dataset_ignore_list
def build_dataset_ignore_list
Definition: cmsHarvester.py:3444
def cmsHarvester.build_dataset_list (   self,
  input_method,
  input_name 
)

def dbs_check_dataset_num_events(self, dataset_name): """Figure out the number of events in each run of this dataset. 

     This is a more efficient way of doing this than calling
    dbs_resolve_number_of_events for each run.         # BUG BUG BUG

This might very well not work at all for spread-out samples. (?)

BUG BUG BUG end

""" # DEBUG DEBUG DEBUG

If we get here DBS should have been set up already.

assert not self.dbs_api is None

DEBUG DEBUG DEBUG end

api = self.dbs_api dbs_query = "find run.number, file.name, file.numevents where dataset = %s " \ "and dataset.status = VALID" % \ dataset_name try: api_result = api.executeQuery(dbs_query) except DbsApiException: msg = "ERROR: Could not execute DBS query" self.logger.fatal(msg) raise Error(msg) try: files_info = {} class Handler(xml.sax.handler.ContentHandler): def startElement(self, name, attrs): if name == "result": run_number = int(attrs["RUNS_RUNNUMBER"]) file_name = str(attrs["FILES_LOGICALFILENAME"]) nevents = int(attrs["FILES_NUMBEROFEVENTS"]) try: files_info[run_number][file_name] = nevents except KeyError: files_info[run_number] = {file_name: nevents} xml.sax.parseString(api_result, Handler()) except SAXParseException: msg = "ERROR: Could not parse DBS server output" self.logger.fatal(msg) raise Error(msg) num_events_catalog = {} for run_number in files_info.keys(): num_events_catalog[run_number] = sum(files_info[run_number].values()) # End of dbs_check_dataset_num_events. return num_events_catalog End of old version. 

Build a list of all datasets to be processed.

Definition at line 3357 of file cmsHarvester.py.

References dbs_resolve_dataset_name(), and list().

3357 
3358  def build_dataset_list(self, input_method, input_name):
3359  """Build a list of all datasets to be processed.
3360 
3361  """
3362 
3363  dataset_names = []
3364 
3365  # It may be, but only for the list of datasets to ignore, that
3366  # the input method and name are None because nothing was
3367  # specified. In that case just an empty list is returned.
3368  if input_method is None:
3369  pass
3370  elif input_method == "dataset":
3371  # Input comes from a dataset name directly on the command
3372  # line. But, this can also contain wildcards so we need
3373  # DBS to translate it conclusively into a list of explicit
3374  # dataset names.
3375  self.logger.info("Asking DBS for dataset names")
3376  dataset_names = self.dbs_resolve_dataset_name(input_name)
3377  elif input_method == "datasetfile":
3378  # In this case a file containing a list of dataset names
3379  # is specified. Still, each line may contain wildcards so
3380  # this step also needs help from DBS.
3381  # NOTE: Lines starting with a `#' are ignored.
3382  self.logger.info("Reading input from list file `%s'" % \
3383  input_name)
3384  try:
3385  listfile = open("/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/harvesting/bin/%s" %input_name, "r")
3386  print "open listfile"
3387  for dataset in listfile:
3388  # Skip empty lines.
3389  dataset_stripped = dataset.strip()
3390  if len(dataset_stripped) < 1:
3391  continue
3392  # Skip lines starting with a `#'.
3393  if dataset_stripped[0] != "#":
3394  dataset_names.extend(self. \
3395  dbs_resolve_dataset_name(dataset_stripped))
3396  listfile.close()
3397  except IOError:
3398  msg = "ERROR: Could not open input list file `%s'" % \
3399  input_name
3400  self.logger.fatal(msg)
3401  raise Error(msg)
3402  else:
3403  # DEBUG DEBUG DEBUG
3404  # We should never get here.
3405  assert False, "Unknown input method `%s'" % input_method
3406  # DEBUG DEBUG DEBUG end
3407 
3408  # Remove duplicates from the dataset list.
3409  # NOTE: There should not be any duplicates in any list coming
3410  # from DBS, but maybe the user provided a list file with less
3411  # care.
3412  dataset_names = list(set(dataset_names))
3413 
3414  # Store for later use.
3415  dataset_names.sort()
3416 
3417  # End of build_dataset_list.
3418  return dataset_names
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
cmsHarvester.build_dataset_list
def build_dataset_list
def dbs_check_dataset_num_events(self, dataset_name): &quot;&quot;"Figure out the number of events in each run ...
Definition: cmsHarvester.py:3357
cmsHarvester.dbs_resolve_dataset_name
def dbs_resolve_dataset_name
Definition: cmsHarvester.py:2418
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def cmsHarvester.build_dataset_use_list (   self) 
Build a list of datasets to process.

Definition at line 3421 of file cmsHarvester.py.

3421 
3422  def build_dataset_use_list(self):
3423  """Build a list of datasets to process.
3424 
3425  """
3426 
3427  self.logger.info("Building list of datasets to consider...")
3428 
3429  input_method = self.input_method["datasets"]["use"]
3430  input_name = self.input_name["datasets"]["use"]
3431  dataset_names = self.build_dataset_list(input_method,
3432  input_name)
3433  self.datasets_to_use = dict(zip(dataset_names,
3434  [None] * len(dataset_names)))
3435 
3436  self.logger.info(" found %d dataset(s) to process:" % \
3437  len(dataset_names))
3438  for dataset in dataset_names:
3439  self.logger.info(" `%s'" % dataset)
3440 
3441  # End of build_dataset_use_list.
archive.zip
tuple zip
Definition: archive.py:476
cmsHarvester.build_dataset_use_list
def build_dataset_use_list
Definition: cmsHarvester.py:3421
def cmsHarvester.build_datasets_information (   self) 
Obtain all information on the datasets that we need to run.

Use DBS to figure out all required information on our
datasets, like the run numbers and the GlobalTag. All
information is stored in the datasets_information member
variable.

Definition at line 5322 of file cmsHarvester.py.

5322 
5323  def build_datasets_information(self):
5324  """Obtain all information on the datasets that we need to run.
5325 
5326  Use DBS to figure out all required information on our
5327  datasets, like the run numbers and the GlobalTag. All
5328  information is stored in the datasets_information member
5329  variable.
5330 
5331  """
5332 
5333  # Get a list of runs in the dataset.
5334  # NOTE: The harvesting has to be done run-by-run, so we
5335  # split up datasets based on the run numbers. Strictly
5336  # speaking this is not (yet?) necessary for Monte Carlo
5337  # since all those samples use run number 1. Still, this
5338  # general approach should work for all samples.
5339 
5340  # Now loop over all datasets in the list and process them.
5341  # NOTE: This processing has been split into several loops
5342  # to be easier to follow, sacrificing a bit of efficiency.
5343  self.datasets_information = {}
5344  self.logger.info("Collecting information for all datasets to process")
5345  dataset_names = self.datasets_to_use.keys()
5346  dataset_names.sort()
5347  for dataset_name in dataset_names:
5348 
5349  # Tell the user which dataset: nice with many datasets.
5350  sep_line = "-" * 30
5351  self.logger.info(sep_line)
5352  self.logger.info(" `%s'" % dataset_name)
5353  self.logger.info(sep_line)
5354 
5355  runs = self.dbs_resolve_runs(dataset_name)
5356  self.logger.info(" found %d run(s)" % len(runs))
5357  if len(runs) > 0:
5358  self.logger.debug(" run number(s): %s" % \
5359  ", ".join([str(i) for i in runs]))
5360  else:
5361  # DEBUG DEBUG DEBUG
5362  # This should never happen after the DBS checks.
5363  self.logger.warning(" --> skipping dataset "
5364  "without any runs")
5365  assert False, "Panic: found a dataset without runs " \
5366  "after DBS checks!"
5367  # DEBUG DEBUG DEBUG end
5368 
5369  cmssw_version = self.dbs_resolve_cmssw_version(dataset_name)
5370  self.logger.info(" found CMSSW version `%s'" % cmssw_version)
5371 
5372  # Figure out if this is data or MC.
5373  datatype = self.dbs_resolve_datatype(dataset_name)
5374  self.logger.info(" sample is data or MC? --> %s" % \
5375  datatype)
5376 
5377  ###
5378 
5379  # Try and figure out the GlobalTag to be used.
5380  if self.globaltag is None:
5381  globaltag = self.dbs_resolve_globaltag(dataset_name)
5382  else:
5383  globaltag = self.globaltag
5384 
5385  self.logger.info(" found GlobalTag `%s'" % globaltag)
5386 
5387  # DEBUG DEBUG DEBUG
5388  if globaltag == "":
5389  # Actually we should not even reach this point, after
5390  # our dataset sanity checks.
5391  assert datatype == "data", \
5392  "ERROR Empty GlobalTag for MC dataset!!!"
5393  # DEBUG DEBUG DEBUG end
5394 
5395  ###
5396 
5397  # DEBUG DEBUG DEBUG
5398  #tmp = self.dbs_check_dataset_spread_old(dataset_name)
5399  # DEBUG DEBUG DEBUG end
5400  sites_catalog = self.dbs_check_dataset_spread(dataset_name)
5401 
5402  # Extract the total event counts.
5403  num_events = {}
5404  for run_number in sites_catalog.keys():
5405  num_events[run_number] = sites_catalog \
5406  [run_number]["all_sites"]
5407  del sites_catalog[run_number]["all_sites"]
5408 
5409  # Extract the information about whether or not datasets
5410  # are mirrored.
5411  mirror_catalog = {}
5412  for run_number in sites_catalog.keys():
5413  mirror_catalog[run_number] = sites_catalog \
5414  [run_number]["mirrored"]
5415  del sites_catalog[run_number]["mirrored"]
5416 
5417  # BUG BUG BUG
5418  # I think I could now get rid of that and just fill the
5419  # "sites" entry with the `inverse' of this
5420  # num_events_catalog(?).
5421  #num_sites = self.dbs_resolve_dataset_number_of_sites(dataset_name)
5422  #sites_catalog = self.dbs_check_dataset_spread(dataset_name)
5423  #sites_catalog = dict(zip(num_events_catalog.keys(),
5424  # [[j for i in num_events_catalog.values() for j in i.keys()]]))
5425  # BUG BUG BUG end
cmsHarvester.build_datasets_information
def build_datasets_information
Definition: cmsHarvester.py:5322
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def cmsHarvester.build_runs_ignore_list (   self) 
Build a list of runs to ignore.

NOTE: We should always have a list of runs to process, but
it may be that we don't have a list of runs to ignore.

Definition at line 3542 of file cmsHarvester.py.

3542 
3543  def build_runs_ignore_list(self):
3544  """Build a list of runs to ignore.
3545 
3546  NOTE: We should always have a list of runs to process, but
3547  it may be that we don't have a list of runs to ignore.
3548 
3549  """
3550 
3551  self.logger.info("Building list of runs to ignore...")
3552 
3553  input_method = self.input_method["runs"]["ignore"]
3554  input_name = self.input_name["runs"]["ignore"]
3555  runs = self.build_runs_list(input_method, input_name)
3556  self.runs_to_ignore = dict(zip(runs, [None] * len(runs)))
3557 
3558  self.logger.info(" found %d run(s) to ignore:" % \
3559  len(runs))
3560  if len(runs) > 0:
3561  self.logger.info(" %s" % ", ".join([str(i) for i in runs]))
3562 
3563  # End of build_runs_ignore_list().
cmsHarvester.build_runs_ignore_list
def build_runs_ignore_list
Definition: cmsHarvester.py:3542
archive.zip
tuple zip
Definition: archive.py:476
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def cmsHarvester.build_runs_list (   self,
  input_method,
  input_name 
)

Definition at line 3470 of file cmsHarvester.py.

References list().

3470 
3471  def build_runs_list(self, input_method, input_name):
3472 
3473  runs = []
3474 
3475  # A list of runs (either to use or to ignore) is not
3476  # required. This protects against `empty cases.'
3477  if input_method is None:
3478  pass
3479  elif input_method == "runs":
3480  # A list of runs was specified directly from the command
3481  # line.
3482  self.logger.info("Reading list of runs from the " \
3483  "command line")
3484  runs.extend([int(i.strip()) \
3485  for i in input_name.split(",") \
3486  if len(i.strip()) > 0])
3487  elif input_method == "runslistfile":
3488  # We were passed a file containing a list of runs.
3489  self.logger.info("Reading list of runs from file `%s'" % \
3490  input_name)
3491  try:
3492  listfile = open(input_name, "r")
3493  for run in listfile:
3494  # Skip empty lines.
3495  run_stripped = run.strip()
3496  if len(run_stripped) < 1:
3497  continue
3498  # Skip lines starting with a `#'.
3499  if run_stripped[0] != "#":
3500  runs.append(int(run_stripped))
3501  listfile.close()
3502  except IOError:
3503  msg = "ERROR: Could not open input list file `%s'" % \
3504  input_name
3505  self.logger.fatal(msg)
3506  raise Error(msg)
3507 
3508  else:
3509  # DEBUG DEBUG DEBUG
3510  # We should never get here.
3511  assert False, "Unknown input method `%s'" % input_method
3512  # DEBUG DEBUG DEBUG end
3513 
3514  # Remove duplicates, sort and done.
3515  runs = list(set(runs))
3516 
3517  # End of build_runs_list().
3518  return runs
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def cmsHarvester.build_runs_use_list (   self) 
Build a list of runs to process.

Definition at line 3521 of file cmsHarvester.py.

3521 
3522  def build_runs_use_list(self):
3523  """Build a list of runs to process.
3524 
3525  """
3526 
3527  self.logger.info("Building list of runs to consider...")
3528 
3529  input_method = self.input_method["runs"]["use"]
3530  input_name = self.input_name["runs"]["use"]
3531  runs = self.build_runs_list(input_method, input_name)
3532  self.runs_to_use = dict(zip(runs, [None] * len(runs)))
3533 
3534  self.logger.info(" found %d run(s) to process:" % \
3535  len(runs))
3536  if len(runs) > 0:
3537  self.logger.info(" %s" % ", ".join([str(i) for i in runs]))
3538 
3539  # End of build_runs_list().
cmsHarvester.build_runs_use_list
def build_runs_use_list
Definition: cmsHarvester.py:3521
archive.zip
tuple zip
Definition: archive.py:476
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def cmsHarvester.check_cmssw (   self) 
Check if CMSSW is setup.

Definition at line 2332 of file cmsHarvester.py.

2332 
2333  def check_cmssw(self):
2334  """Check if CMSSW is setup.
2335 
2336  """
2337 
2338  # Try to access the CMSSW_VERSION environment variable. If
2339  # it's something useful we consider CMSSW to be set up
2340  # properly. Otherwise we raise an error.
2341  cmssw_version = os.getenv("CMSSW_VERSION")
2342  if cmssw_version is None:
2343  self.logger.fatal("It seems CMSSW is not setup...")
2344  self.logger.fatal("($CMSSW_VERSION is empty)")
2345  raise Error("ERROR: CMSSW needs to be setup first!")
2346 
2347  self.cmssw_version = cmssw_version
2348  self.logger.info("Found CMSSW version %s properly set up" % \
2349  self.cmssw_version)
2350 
2351  # End of check_cmsssw.
2352  return True
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
def cmsHarvester.check_dataset_list (   self) 
Check list of dataset names for impossible ones.

Two kinds of checks are done:
- Checks for things that do not make sense. These lead to
  errors and skipped datasets.
- Sanity checks. For these warnings are issued but the user is
  considered to be the authoritative expert.

Checks performed:
- The CMSSW version encoded in the dataset name should match
  self.cmssw_version. This is critical.
- There should be some events in the dataset/run. This is
  critical in the sense that CRAB refuses to create jobs for
  zero events. And yes, this does happen in practice. E.g. the
  reprocessed CRAFT08 datasets contain runs with zero events.
- A cursory check is performed to see if the harvesting type
  makes sense for the data type. This should prevent the user
  from inadvertently running RelVal for data.
- It is not possible to run single-step harvesting jobs on
  samples that are not fully contained at a single site.
- Each dataset/run has to be available at at least one site.

Definition at line 3795 of file cmsHarvester.py.

3795 
3796  def check_dataset_list(self):
3797  """Check list of dataset names for impossible ones.
3798 
3799  Two kinds of checks are done:
3800  - Checks for things that do not make sense. These lead to
3801  errors and skipped datasets.
3802  - Sanity checks. For these warnings are issued but the user is
3803  considered to be the authoritative expert.
3804 
3805  Checks performed:
3806  - The CMSSW version encoded in the dataset name should match
3807  self.cmssw_version. This is critical.
3808  - There should be some events in the dataset/run. This is
3809  critical in the sense that CRAB refuses to create jobs for
3810  zero events. And yes, this does happen in practice. E.g. the
3811  reprocessed CRAFT08 datasets contain runs with zero events.
3812  - A cursory check is performed to see if the harvesting type
3813  makes sense for the data type. This should prevent the user
3814  from inadvertently running RelVal for data.
3815  - It is not possible to run single-step harvesting jobs on
3816  samples that are not fully contained at a single site.
3817  - Each dataset/run has to be available at at least one site.
3818 
3819  """
3820 
3821  self.logger.info("Performing sanity checks on dataset list...")
3822 
3823  dataset_names_after_checks = copy.deepcopy(self.datasets_to_use)
3824 
3825  for dataset_name in self.datasets_to_use.keys():
3826 
3827  # Check CMSSW version.
3828  version_from_dataset = self.datasets_information[dataset_name] \
3829  ["cmssw_version"]
3830  if version_from_dataset != self.cmssw_version:
3831  msg = " CMSSW version mismatch for dataset `%s' " \
3832  "(%s vs. %s)" % \
3833  (dataset_name,
3834  self.cmssw_version, version_from_dataset)
3835  if self.force_running:
3836  # Expert mode: just warn, then continue.
3837  self.logger.warning("%s " \
3838  "--> `force mode' active: " \
3839  "run anyway" % msg)
3840  else:
3841  del dataset_names_after_checks[dataset_name]
3842  self.logger.warning("%s " \
3843  "--> skipping" % msg)
3844  continue
3845 
3846  ###
3847 
3848  # Check that the harvesting type makes sense for the
3849  # sample. E.g. normally one would not run the DQMOffline
3850  # harvesting on Monte Carlo.
3851  # TODO TODO TODO
3852  # This should be further refined.
3853  suspicious = False
3854  datatype = self.datasets_information[dataset_name]["datatype"]
3855  if datatype == "data":
3856  # Normally only DQM harvesting is run on data.
3857  if self.harvesting_type != "DQMOffline":
3858  suspicious = True
3859  elif datatype == "mc":
3860  if self.harvesting_type == "DQMOffline":
3861  suspicious = True
3862  else:
3863  # Doh!
3864  assert False, "ERROR Impossible data type `%s' " \
3865  "for dataset `%s'" % \
3866  (datatype, dataset_name)
3867  if suspicious:
3868  msg = " Normally one does not run `%s' harvesting " \
3869  "on %s samples, are you sure?" % \
3870  (self.harvesting_type, datatype)
3871  if self.force_running:
3872  self.logger.warning("%s " \
3873  "--> `force mode' active: " \
3874  "run anyway" % msg)
3875  else:
3876  del dataset_names_after_checks[dataset_name]
3877  self.logger.warning("%s " \
3878  "--> skipping" % msg)
3879  continue
3880 
3881  # TODO TODO TODO end
3882 
3883  ###
3884 
3885  # BUG BUG BUG
3886  # For the moment, due to a problem with DBS, I cannot
3887  # figure out the GlobalTag for data by myself. (For MC
3888  # it's no problem.) This means that unless a GlobalTag was
3889  # specified from the command line, we will have to skip
3890  # any data datasets.
3891 
3892  if datatype == "data":
3893  if self.globaltag is None:
3894  msg = "For data datasets (like `%s') " \
3895  "we need a GlobalTag" % \
3896  dataset_name
3897  del dataset_names_after_checks[dataset_name]
3898  self.logger.warning("%s " \
3899  "--> skipping" % msg)
3900  continue
3901 
3902  # BUG BUG BUG end
3903 
3904  ###
3905 
3906  # Check if the GlobalTag exists and (if we're using
3907  # reference histograms) if it's ready to be used with
3908  # reference histograms.
3909  globaltag = self.datasets_information[dataset_name]["globaltag"]
3910  if not globaltag in self.globaltag_check_cache:
3911  if self.check_globaltag(globaltag):
3912  self.globaltag_check_cache.append(globaltag)
3913  else:
3914  msg = "Something is wrong with GlobalTag `%s' " \
3915  "used by dataset `%s'!" % \
3916  (globaltag, dataset_name)
3917  if self.use_ref_hists:
3918  msg += "\n(Either it does not exist or it " \
3919  "does not contain the required key to " \
3920  "be used with reference histograms.)"
3921  else:
3922  msg += "\n(It probably just does not exist.)"
3923  self.logger.fatal(msg)
3924  raise Usage(msg)
3925 
3926  ###
3927 
3928  # Require that each run is available at least somewhere.
3929  runs_without_sites = [i for (i, j) in \
3930  self.datasets_information[dataset_name] \
3931  ["sites"].items() \
3932  if len(j) < 1 and \
3933  i in self.datasets_to_use[dataset_name]]
3934  if len(runs_without_sites) > 0:
3935  for run_without_sites in runs_without_sites:
3936  try:
3937  dataset_names_after_checks[dataset_name].remove(run_without_sites)
3938  except KeyError:
3939  pass
3940  self.logger.warning(" removed %d unavailable run(s) " \
3941  "from dataset `%s'" % \
3942  (len(runs_without_sites), dataset_name))
3943  self.logger.debug(" (%s)" % \
3944  ", ".join([str(i) for i in \
3945  runs_without_sites]))
3946 
3947  ###
3948 
3949  # Unless we're running two-step harvesting: only allow
3950  # samples located on a single site.
3951  if not self.harvesting_mode == "two-step":
3952  for run_number in self.datasets_to_use[dataset_name]:
# DEBUG DEBUG DEBUG
cmsHarvester.check_dataset_list
def check_dataset_list
Definition: cmsHarvester.py:3795
cmsHarvester.Usage
Helper class: Usage exception.
Definition: cmsHarvester.py:128
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def cmsHarvester.check_dbs (   self) 
Check if DBS is setup.

Definition at line 2355 of file cmsHarvester.py.

2355 
2356  def check_dbs(self):
2357  """Check if DBS is setup.
2358 
2359  """
2360 
2361  # Try to access the DBSCMD_HOME environment variable. If this
2362  # looks useful we consider DBS to be set up
2363  # properly. Otherwise we raise an error.
2364  dbs_home = os.getenv("DBSCMD_HOME")
2365  if dbs_home is None:
2366  self.logger.fatal("It seems DBS is not setup...")
2367  self.logger.fatal(" $DBSCMD_HOME is empty")
2368  raise Error("ERROR: DBS needs to be setup first!")
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
def cmsHarvester.check_globaltag (   self,
  globaltag = None 
) 
Check if globaltag exists.

Check if globaltag exists as GlobalTag in the database given
by self.frontier_connection_name['globaltag']. If globaltag is
None, self.globaltag is used instead.

If we're going to use reference histograms this method also
checks for the existence of the required key in the GlobalTag.

Definition at line 4502 of file cmsHarvester.py.

4502 
4503  def check_globaltag(self, globaltag=None):
4504  """Check if globaltag exists.
4505 
4506  Check if globaltag exists as GlobalTag in the database given
4507  by self.frontier_connection_name['globaltag']. If globaltag is
4508  None, self.globaltag is used instead.
4509 
4510  If we're going to use reference histograms this method also
4511  checks for the existence of the required key in the GlobalTag.
4512 
4513  """
4514 
4515  if globaltag is None:
4516  globaltag = self.globaltag
4517 
4518  # All GlobalTags should end in `::All', right?
4519  if globaltag.endswith("::All"):
4520  globaltag = globaltag[:-5]
4521 
4522  connect_name = self.frontier_connection_name["globaltag"]
4523  # BUG BUG BUG
4524  # There is a bug in cmscond_tagtree_list: some magic is
4525  # missing from the implementation requiring one to specify
4526  # explicitly the name of the squid to connect to. Since the
4527  # cmsHarvester can only be run from the CERN network anyway,
4528  # cmsfrontier:8000 is hard-coded in here. Not nice but it
4529  # works.
4530  connect_name = connect_name.replace("frontier://",
4531  "frontier://cmsfrontier:8000/")
4532  # BUG BUG BUG end
4533  connect_name += self.db_account_name_cms_cond_globaltag()
4534 
4535  tag_exists = self.check_globaltag_exists(globaltag, connect_name)
4536 
4537  #----------
4538 
4539  tag_contains_ref_hist_key = False
4540  if self.use_ref_hists and tag_exists:
4541  # Check for the key required to use reference histograms.
4542  tag_contains_ref_hist_key = self.check_globaltag_contains_ref_hist_key(globaltag, connect_name)
4543 
4544  #----------
4545 
4546  if self.use_ref_hists:
4547  ret_val = tag_exists and tag_contains_ref_hist_key
4548  else:
4549  ret_val = tag_exists
4550 
4551  #----------
4552 
4553  # End of check_globaltag.
4554  return ret_val
def cmsHarvester.check_globaltag_contains_ref_hist_key (   self,
  globaltag,
  connect_name 
) 
Check if globaltag contains the required RefHistos key.

Definition at line 4599 of file cmsHarvester.py.

4599 
4600  def check_globaltag_contains_ref_hist_key(self, globaltag, connect_name):
4601  """Check if globaltag contains the required RefHistos key.
4602 
4603  """
4604 
4605  # Check for the key required to use reference histograms.
4606  tag_contains_key = None
4607  ref_hist_key = "RefHistos"
4608  self.logger.info("Checking existence of reference " \
4609  "histogram key `%s' in GlobalTag `%s'" % \
4610  (ref_hist_key, globaltag))
4611  self.logger.debug(" (Using database connection `%s')" % \
4612  connect_name)
4613  cmd = "cmscond_tagtree_list -c %s -T %s -n %s" % \
4614  (connect_name, globaltag, ref_hist_key)
4615  (status, output) = commands.getstatusoutput(cmd)
4616  if status != 0 or \
4617  output.find("error") > -1:
4618  msg = "Could not check existence of key `%s'" % \
4619  (ref_hist_key, connect_name)
4620  self.logger.fatal(msg)
4621  self.logger.debug("Command used:")
4622  self.logger.debug(" %s" % cmd)
4623  self.logger.debug("Output received:")
4624  self.logger.debug(" %s" % output)
4625  raise Error(msg)
4626  if len(output) < 1:
4627  self.logger.debug("Required key for use of reference " \
4628  "histograms `%s' does not exist " \
4629  "in GlobalTag `%s':" % \
4630  (ref_hist_key, globaltag))
4631  self.logger.debug("Output received:")
4632  self.logger.debug(output)
4633  tag_contains_key = False
4634  else:
4635  tag_contains_key = True
4636 
4637  self.logger.info(" GlobalTag contains `%s' key? -> %s" % \
4638  (ref_hist_key, tag_contains_key))
4639 
4640  # End of check_globaltag_contains_ref_hist_key.
4641  return tag_contains_key
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
cmsHarvester.check_globaltag_contains_ref_hist_key
def check_globaltag_contains_ref_hist_key
Definition: cmsHarvester.py:4599
def cmsHarvester.check_globaltag_exists (   self,
  globaltag,
  connect_name 
) 
Check if globaltag exists.

Definition at line 4557 of file cmsHarvester.py.

References split.

4557 
4558  def check_globaltag_exists(self, globaltag, connect_name):
4559  """Check if globaltag exists.
4560 
4561  """
4562 
4563  self.logger.info("Checking existence of GlobalTag `%s'" % \
4564  globaltag)
4565  self.logger.debug(" (Using database connection `%s')" % \
4566  connect_name)
4567 
4568  cmd = "cmscond_tagtree_list -c %s -T %s" % \
4569  (connect_name, globaltag)
4570  (status, output) = commands.getstatusoutput(cmd)
4571  if status != 0 or \
4572  output.find("error") > -1:
4573  msg = "Could not check existence of GlobalTag `%s' in `%s'" % \
4574  (globaltag, connect_name)
4575  if output.find(".ALL_TABLES not found") > -1:
4576  msg = "%s\n" \
4577  "Missing database account `%s'" % \
4578  (msg, output.split(".ALL_TABLES")[0].split()[-1])
4579  self.logger.fatal(msg)
4580  self.logger.debug("Command used:")
4581  self.logger.debug(" %s" % cmd)
4582  self.logger.debug("Output received:")
4583  self.logger.debug(output)
4584  raise Error(msg)
4585  if output.find("does not exist") > -1:
4586  self.logger.debug("GlobalTag `%s' does not exist in `%s':" % \
4587  (globaltag, connect_name))
4588  self.logger.debug("Output received:")
4589  self.logger.debug(output)
4590  tag_exists = False
4591  else:
4592  tag_exists = True
4593  self.logger.info(" GlobalTag exists? -> %s" % tag_exists)
4594 
4595  # End of check_globaltag_exists.
4596  return tag_exists
cmsHarvester.check_globaltag_exists
def check_globaltag_exists
Definition: cmsHarvester.py:4557
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
split
double split
Definition: MVATrainer.cc:139
def cmsHarvester.check_input_status (   self) 
Check completeness and correctness of input information.

Check that all required information has been specified and
that, at least as far as can be easily checked, it makes
sense.

NOTE: This is also where any default values are applied.

Definition at line 2191 of file cmsHarvester.py.

References join().

2191 
2192  def check_input_status(self):
2193  """Check completeness and correctness of input information.
2194 
2195  Check that all required information has been specified and
2196  that, at least as far as can be easily checked, it makes
2197  sense.
2198 
2199  NOTE: This is also where any default values are applied.
2200 
2201  """
2202 
2203  self.logger.info("Checking completeness/correctness of input...")
2204 
2205  # The cmsHarvester does not take (i.e. understand) any
2206  # arguments so there should not be any.
2207  if len(self.args) > 0:
2208  msg = "Sorry but I don't understand `%s'" % \
2209  (" ".join(self.args))
2210  self.logger.fatal(msg)
2211  raise Usage(msg)
2212 
2213  # BUG BUG BUG
2214  # While we wait for some bugs left and right to get fixed, we
2215  # disable two-step.
2216  if self.harvesting_mode == "two-step":
2217  msg = "--------------------\n" \
2218  " Sorry, but for the moment (well, till it works)" \
2219  " the two-step mode has been disabled.\n" \
2220  "--------------------\n"
2221  self.logger.fatal(msg)
2222  raise Error(msg)
2223  # BUG BUG BUG end
2224 
2225  # We need a harvesting method to be specified
2226  if self.harvesting_type is None:
2227  msg = "Please specify a harvesting type"
2228  self.logger.fatal(msg)
2229  raise Usage(msg)
2230  # as well as a harvesting mode.
2231  if self.harvesting_mode is None:
2232  self.harvesting_mode = self.harvesting_mode_default
2233  msg = "No harvesting mode specified --> using default `%s'" % \
2234  self.harvesting_mode
2235  self.logger.warning(msg)
2236  #raise Usage(msg)
2237 
2238  ###
2239 
2240  # We need an input method so we can find the dataset name(s).
2241  if self.input_method["datasets"]["use"] is None:
2242  msg = "Please specify an input dataset name " \
2243  "or a list file name"
2244  self.logger.fatal(msg)
2245  raise Usage(msg)
2246 
2247  # DEBUG DEBUG DEBUG
2248  # If we get here, we should also have an input name.
2249  assert not self.input_name["datasets"]["use"] is None
2250  # DEBUG DEBUG DEBUG end
2251 
2252  ###
2253 
2254  # The same holds for the reference histogram mapping file (if
2255  # we're using references).
2256  if self.use_ref_hists:
2257  if self.ref_hist_mappings_file_name is None:
2258  self.ref_hist_mappings_file_name = self.ref_hist_mappings_file_name_default
2259  msg = "No reference histogram mapping file specified --> " \
2260  "using default `%s'" % \
2261  self.ref_hist_mappings_file_name
2262  self.logger.warning(msg)
2263 
2264  ###
2265 
2266  # We need to know where to put the stuff (okay, the results)
2267  # on CASTOR.
2268  if self.castor_base_dir is None:
2269  self.castor_base_dir = self.castor_base_dir_default
2270  msg = "No CASTOR area specified -> using default `%s'" % \
2271  self.castor_base_dir
2272  self.logger.warning(msg)
2273  #raise Usage(msg)
2274 
2275  # Only the CERN CASTOR area is supported.
2276  if not self.castor_base_dir.startswith(self.castor_prefix):
2277  msg = "CASTOR area does not start with `%s'" % \
2278  self.castor_prefix
2279  self.logger.fatal(msg)
2280  if self.castor_base_dir.find("castor") > -1 and \
2281  not self.castor_base_dir.find("cern.ch") > -1:
2282  self.logger.fatal("Only CERN CASTOR is supported")
2283  raise Usage(msg)
2284 
2285  ###
2286 
2287  # TODO TODO TODO
2288  # This should be removed in the future, once I find out how to
2289  # get the config file used to create a given dataset from DBS.
2290 
2291  # For data we need to have a GlobalTag. (For MC we can figure
2292  # it out by ourselves.)
2293  if self.globaltag is None:
2294  self.logger.warning("No GlobalTag specified. This means I cannot")
2295  self.logger.warning("run on data, only on MC.")
2296  self.logger.warning("I will skip all data datasets.")
2297 
2298  # TODO TODO TODO end
2299 
2300  # Make sure the GlobalTag ends with `::All'.
2301  if not self.globaltag is None:
2302  if not self.globaltag.endswith("::All"):
2303  self.logger.warning("Specified GlobalTag `%s' does " \
2304  "not end in `::All' --> " \
2305  "appending this missing piece" % \
2306  self.globaltag)
2307  self.globaltag = "%s::All" % self.globaltag
2308 
2309  ###
2310 
2311  # Dump some info about the Frontier connections used.
2312  for (key, value) in self.frontier_connection_name.iteritems():
2313  frontier_type_str = "unknown"
2314  if key == "globaltag":
2315  frontier_type_str = "the GlobalTag"
2316  elif key == "refhists":
2317  frontier_type_str = "the reference histograms"
2318  non_str = None
2319  if self.frontier_connection_overridden[key] == True:
2320  non_str = "non-"
2321  else:
2322  non_str = ""
2323  self.logger.info("Using %sdefault Frontier " \
2324  "connection for %s: `%s'" % \
2325  (non_str, frontier_type_str, value))
2326 
2327  ###
2328 
2329  # End of check_input_status.
cmsHarvester.check_input_status
def check_input_status
Definition: cmsHarvester.py:2191
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
cmsHarvester.Usage
Helper class: Usage exception.
Definition: cmsHarvester.py:128
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def cmsHarvester.check_ref_hist_mappings (   self) 
Make sure all necessary reference histograms exist.

Check that for each of the datasets to be processed a
reference histogram is specified and that that histogram
exists in the database.

NOTE: There's a little complication here. Since this whole
thing was designed to allow (in principle) harvesting of both
data and MC datasets in one go, we need to be careful to check
the availability fof reference mappings only for those
datasets that need it.

Definition at line 5282 of file cmsHarvester.py.

5282 
5283  def check_ref_hist_mappings(self):
5284  """Make sure all necessary reference histograms exist.
5285 
5286  Check that for each of the datasets to be processed a
5287  reference histogram is specified and that that histogram
5288  exists in the database.
5289 
5290  NOTE: There's a little complication here. Since this whole
5291  thing was designed to allow (in principle) harvesting of both
5292  data and MC datasets in one go, we need to be careful to check
5293  the availability fof reference mappings only for those
5294  datasets that need it.
5295 
5296  """
5297 
5298  self.logger.info("Checking reference histogram mappings")
5299 
5300  for dataset_name in self.datasets_to_use:
5301  try:
5302  ref_hist_name = self.ref_hist_mappings[dataset_name]
5303  except KeyError:
5304  msg = "ERROR: No reference histogram mapping found " \
5305  "for dataset `%s'" % \
5306  dataset_name
5307  self.logger.fatal(msg)
5308  raise Error(msg)
5309 
5310  if not self.check_ref_hist_tag(ref_hist_name):
5311  msg = "Reference histogram tag `%s' " \
5312  "(used for dataset `%s') does not exist!" % \
5313  (ref_hist_name, dataset_name)
5314  self.logger.fatal(msg)
5315  raise Usage(msg)
5316 
5317  self.logger.info(" Done checking reference histogram mappings.")
5318 
5319  # End of check_ref_hist_mappings.
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
cmsHarvester.Usage
Helper class: Usage exception.
Definition: cmsHarvester.py:128
cmsHarvester.check_ref_hist_mappings
def check_ref_hist_mappings
Definition: cmsHarvester.py:5282
def cmsHarvester.check_ref_hist_tag (   self,
  tag_name 
) 
Check the existence of tag_name in database connect_name.

Check if tag_name exists as a reference histogram tag in the
database given by self.frontier_connection_name['refhists'].

Definition at line 4644 of file cmsHarvester.py.

References join().

4644 
4645  def check_ref_hist_tag(self, tag_name):
4646  """Check the existence of tag_name in database connect_name.
4647 
4648  Check if tag_name exists as a reference histogram tag in the
4649  database given by self.frontier_connection_name['refhists'].
4650 
4651  """
4652 
4653  connect_name = self.frontier_connection_name["refhists"]
4654  connect_name += self.db_account_name_cms_cond_dqm_summary()
4655 
4656  self.logger.debug("Checking existence of reference " \
4657  "histogram tag `%s'" % \
4658  tag_name)
4659  self.logger.debug(" (Using database connection `%s')" % \
4660  connect_name)
4661 
4662  cmd = "cmscond_list_iov -c %s" % \
4663  connect_name
4664  (status, output) = commands.getstatusoutput(cmd)
4665  if status != 0:
4666  msg = "Could not check existence of tag `%s' in `%s'" % \
4667  (tag_name, connect_name)
4668  self.logger.fatal(msg)
4669  self.logger.debug("Command used:")
4670  self.logger.debug(" %s" % cmd)
4671  self.logger.debug("Output received:")
4672  self.logger.debug(output)
4673  raise Error(msg)
4674  if not tag_name in output.split():
4675  self.logger.debug("Reference histogram tag `%s' " \
4676  "does not exist in `%s'" % \
4677  (tag_name, connect_name))
4678  self.logger.debug(" Existing tags: `%s'" % \
4679  "', `".join(output.split()))
4680  tag_exists = False
4681  else:
4682  tag_exists = True
4683  self.logger.debug(" Reference histogram tag exists? " \
4684  "-> %s" % tag_exists)
4685 
4686  # End of check_ref_hist_tag.
4687  return tag_exists
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
cmsHarvester.check_ref_hist_tag
def check_ref_hist_tag
Definition: cmsHarvester.py:4644
def cmsHarvester.create_and_check_castor_dir (   self,
  castor_dir 
) 
Check existence of the give CASTOR dir, if necessary create
it.

Some special care has to be taken with several things like
setting the correct permissions such that CRAB can store the
output results. Of course this means that things like
/castor/cern.ch/ and user/j/ have to be recognised and treated
properly.

NOTE: Only CERN CASTOR area (/castor/cern.ch/) supported for
the moment.

NOTE: This method uses some slightly tricky caching to make
sure we don't keep over and over checking the same base paths.

Definition at line 1488 of file cmsHarvester.py.

References spr.find().

1488 
1489  def create_and_check_castor_dir(self, castor_dir):
1490  """Check existence of the give CASTOR dir, if necessary create
1491  it.
1492 
1493  Some special care has to be taken with several things like
1494  setting the correct permissions such that CRAB can store the
1495  output results. Of course this means that things like
1496  /castor/cern.ch/ and user/j/ have to be recognised and treated
1497  properly.
1498 
1499  NOTE: Only CERN CASTOR area (/castor/cern.ch/) supported for
1500  the moment.
1501 
1502  NOTE: This method uses some slightly tricky caching to make
1503  sure we don't keep over and over checking the same base paths.
1504 
1505  """
1506 
1507  ###
1508 
1509  # Local helper function to fully split a path into pieces.
1510  def split_completely(path):
1511  (parent_path, name) = os.path.split(path)
1512  if name == "":
1513  return (parent_path, )
1514  else:
1515  return split_completely(parent_path) + (name, )
1516 
1517  ###
1518 
1519  # Local helper function to check rfio (i.e. CASTOR)
1520  # directories.
1521  def extract_permissions(rfstat_output):
1522  """Parse the output from rfstat and return the
1523  5-digit permissions string."""
1524 
1525  permissions_line = [i for i in output.split("\n") \
1526  if i.lower().find("protection") > -1]
1527  regexp = re.compile(".*\(([0123456789]{5})\).*")
1528  match = regexp.search(rfstat_output)
1529  if not match or len(match.groups()) != 1:
1530  msg = "Could not extract permissions " \
1531  "from output: %s" % rfstat_output
1532  self.logger.fatal(msg)
1533  raise Error(msg)
1534  permissions = match.group(1)
1535 
1536  # End of extract_permissions.
1537  return permissions
1538 
1539  ###
1540 
1541  # These are the pieces of CASTOR directories that we do not
1542  # want to touch when modifying permissions.
1543 
1544  # NOTE: This is all a bit involved, basically driven by the
1545  # fact that one wants to treat the `j' directory of
1546  # `/castor/cern.ch/user/j/jhegeman/' specially.
1547  # BUG BUG BUG
1548  # This should be simplified, for example by comparing to the
1549  # CASTOR prefix or something like that.
1550  # BUG BUG BUG end
1551  castor_paths_dont_touch = {
1552  0: ["/", "castor", "cern.ch", "cms", "store", "temp",
1553  "dqm", "offline", "user"],
1554  -1: ["user", "store"]
1555  }
1556 
1557  self.logger.debug("Checking CASTOR path `%s'" % castor_dir)
1558 
1559  ###
1560 
1561  # First we take the full CASTOR path apart.
1562  castor_path_pieces = split_completely(castor_dir)
1563 
1564  # Now slowly rebuild the CASTOR path and see if a) all
1565  # permissions are set correctly and b) the final destination
1566  # exists.
1567  path = ""
1568  check_sizes = castor_paths_dont_touch.keys()
1569  check_sizes.sort()
1570  len_castor_path_pieces = len(castor_path_pieces)
1571  for piece_index in xrange (len_castor_path_pieces):
1572  skip_this_path_piece = False
piece = castor_path_pieces[piece_index]
spr::find
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:7
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
cmsHarvester.create_and_check_castor_dir
def create_and_check_castor_dir
Definition: cmsHarvester.py:1488
def cmsHarvester.create_and_check_castor_dirs (   self) 
Make sure all required CASTOR output dirs exist.

This checks the CASTOR base dir specified by the user as well
as all the subdirs required by the current set of jobs.

Definition at line 1428 of file cmsHarvester.py.

References list(), and bookConverter.max.

1428 
1429  def create_and_check_castor_dirs(self):
1430  """Make sure all required CASTOR output dirs exist.
1431 
1432  This checks the CASTOR base dir specified by the user as well
1433  as all the subdirs required by the current set of jobs.
1434 
1435  """
1436 
1437  self.logger.info("Checking (and if necessary creating) CASTOR " \
1438  "output area(s)...")
1439 
1440  # Call the real checker method for the base dir.
1441  self.create_and_check_castor_dir(self.castor_base_dir)
1442 
1443  # Now call the checker for all (unique) subdirs.
1444  castor_dirs = []
1445  for (dataset_name, runs) in self.datasets_to_use.iteritems():
1446 
1447  for run in runs:
1448  castor_dirs.append(self.datasets_information[dataset_name] \
1449  ["castor_path"][run])
1450  castor_dirs_unique = list(set(castor_dirs))
1451  castor_dirs_unique.sort()
1452  # This can take some time. E.g. CRAFT08 has > 300 runs, each
1453  # of which will get a new directory. So we show some (rough)
1454  # info in between.
1455  ndirs = len(castor_dirs_unique)
1456  step = max(ndirs / 10, 1)
1457  for (i, castor_dir) in enumerate(castor_dirs_unique):
1458  if (i + 1) % step == 0 or \
1459  (i + 1) == ndirs:
1460  self.logger.info(" %d/%d" % \
1461  (i + 1, ndirs))
1462  self.create_and_check_castor_dir(castor_dir)
1463 
1464  # Now check if the directory is empty. If (an old version
1465  # of) the output file already exists CRAB will run new
1466  # jobs but never copy the results back. We assume the user
1467  # knows what they are doing and only issue a warning in
1468  # case the directory is not empty.
1469  self.logger.debug("Checking if path `%s' is empty" % \
1470  castor_dir)
1471  cmd = "rfdir %s" % castor_dir
1472  (status, output) = commands.getstatusoutput(cmd)
1473  if status != 0:
1474  msg = "Could not access directory `%s'" \
1475  " !!! This is bad since I should have just" \
1476  " created it !!!" % castor_dir
1477  self.logger.fatal(msg)
1478  raise Error(msg)
1479  if len(output) > 0:
1480  self.logger.warning("Output directory `%s' is not empty:" \
1481  " new jobs will fail to" \
1482  " copy back output" % \
1483  castor_dir)
1484 
1485  # End of create_and_check_castor_dirs.
cmsHarvester.create_and_check_castor_dirs
def create_and_check_castor_dirs
Definition: cmsHarvester.py:1428
cmsHarvester.Error
Helper class: Error exception.
Definition: cmsHarvester.py:139
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def cmsHarvester.create_castor_path_name_common (   self,
  dataset_name 
) 
Build the common part of the output path to be used on
CASTOR.

This consists of the CASTOR area base path specified by the
user and a piece depending on the data type (data vs. MC), the
harvesting type and the dataset name followed by a piece
containing the run number and event count. (See comments in
create_castor_path_name_special for details.) This method
creates the common part, without run number and event count.

Definition at line 1324 of file cmsHarvester.py.

References create_castor_path_name_special(), and python.rootplot.root2matplotlib.replace().

1324 
1325  def create_castor_path_name_common(self, dataset_name):
1326  """Build the common part of the output path to be used on
1327  CASTOR.
1328 
1329  This consists of the CASTOR area base path specified by the
1330  user and a piece depending on the data type (data vs. MC), the
1331  harvesting type and the dataset name followed by a piece
1332  containing the run number and event count. (See comments in
1333  create_castor_path_name_special for details.) This method
1334  creates the common part, without run number and event count.
1335 
1336  """
1337 
1338  castor_path = self.castor_base_dir
1339 
1340  ###
1341 
1342  # The data type: data vs. mc.
1343  datatype = self.datasets_information[dataset_name]["datatype"]
1344  datatype = datatype.lower()
1345  castor_path = os.path.join(castor_path, datatype)
1346 
1347  # The harvesting type.
1348  harvesting_type = self.harvesting_type
1349  harvesting_type = harvesting_type.lower()
1350  castor_path = os.path.join(castor_path, harvesting_type)
1351 
1352  # The CMSSW release version (only the `digits'). Note that the
1353  # CMSSW version used here is the version used for harvesting,
1354  # not the one from the dataset. This does make the results
1355  # slightly harder to find. On the other hand it solves
1356  # problems in case one re-harvests a given dataset with a
1357  # different CMSSW version, which would lead to ambiguous path
1358  # names. (Of course for many cases the harvesting is done with
1359  # the same CMSSW version the dataset was created with.)
1360  release_version = self.cmssw_version
1361  release_version = release_version.lower(). \
1362  replace("cmssw", ""). \
1363  strip("_")
1364  castor_path = os.path.join(castor_path, release_version)
1365 
1366  # The dataset name.
1367  dataset_name_escaped = self.escape_dataset_name(dataset_name)
1368  castor_path = os.path.join(castor_path, dataset_name_escaped)
1369 
1370  ###
1371 
1372  castor_path = os.path.normpath(castor_path)
1373 
1374  # End of create_castor_path_name_common.
1375  return castor_path
cmsHarvester.create_castor_path_name_common
def create_castor_path_name_common
Definition: cmsHarvester.py:1324
def cmsHarvester.create_castor_path_name_special (   self,
  dataset_name,
  run_number,
  castor_path_common 
) 
Create the specialised part of the CASTOR output dir name.

NOTE: To avoid clashes with `incremental harvesting'
(re-harvesting when a dataset grows) we have to include the
event count in the path name. The underlying `problem' is that
CRAB does not overwrite existing output files so if the output
file already exists CRAB will fail to copy back the output.

NOTE: It's not possible to create different kinds of
harvesting jobs in a single call to this tool. However, in
principle it could be possible to create both data and MC jobs
in a single go.

NOTE: The number of events used in the path name is the
_total_ number of events in the dataset/run at the time of
harvesting. If we're doing partial harvesting the final
results will reflect lower statistics. This is a) the easiest
to code and b) the least likely to lead to confusion if
someone ever decides to swap/copy around file blocks between
sites.

Definition at line 1380 of file cmsHarvester.py.

Referenced by create_castor_path_name_common().

1380 
1381  castor_path_common):
1382  """Create the specialised part of the CASTOR output dir name.
1383 
1384  NOTE: To avoid clashes with `incremental harvesting'
1385  (re-harvesting when a dataset grows) we have to include the
1386  event count in the path name. The underlying `problem' is that
1387  CRAB does not overwrite existing output files so if the output
1388  file already exists CRAB will fail to copy back the output.
1389 
1390  NOTE: It's not possible to create different kinds of
1391  harvesting jobs in a single call to this tool. However, in
1392  principle it could be possible to create both data and MC jobs
1393  in a single go.
1394 
1395  NOTE: The number of events used in the path name is the
1396  _total_ number of events in the dataset/run at the time of
1397  harvesting. If we're doing partial harvesting the final
1398  results will reflect lower statistics. This is a) the easiest
1399  to code and b) the least likely to lead to confusion if
1400  someone ever decides to swap/copy around file blocks between
1401  sites.
1402 
1403  """
1404 
1405  castor_path = castor_path_common
1406 
1407  ###
1408 
1409  # The run number part.
1410  castor_path = os.path.join(castor_path, "run_%d" % run_number)
1411 
1412  ###
1413 
1414  # The event count (i.e. the number of events we currently see
1415  # for this dataset).
1416  #nevents = self.datasets_information[dataset_name] \
1417  # ["num_events"][run_number]
1418  castor_path = os.path.join(castor_path, "nevents")
1419 
1420  ###
1421 
1422  castor_path = os.path.normpath(castor_path)
1423 
1424  # End of create_castor_path_name_special.
1425  return castor_path
def cmsHarvester.create_config_file_name (   self,
  dataset_name,
  run_number 
) 
Generate the name of the configuration file to be run by
CRAB.

Depending on the harvesting mode (single-step or two-step)
this is the name of the real harvesting configuration or the
name of the first-step ME summary extraction configuration.

Definition at line 4065 of file cmsHarvester.py.

4065 
4066  def create_config_file_name(self, dataset_name, run_number):
4067  """Generate the name of the configuration file to be run by
4068  CRAB.
4069 
4070  Depending on the harvesting mode (single-step or two-step)
4071  this is the name of the real harvesting configuration or the
4072  name of the first-step ME summary extraction configuration.
4073 
4074  """
4075 
4076  if self.harvesting_mode == "single-step":
4077  config_file_name = self.create_harvesting_config_file_name(dataset_name)
4078  elif self.harvesting_mode == "single-step-allow-partial":
config_file_name = self.create_harvesting_config_file_name(dataset_name)
cmsHarvester.create_config_file_name
def create_config_file_name
Definition: cmsHarvester.py:4065
def cmsHarvester.create_crab_config (   self) 
Create a CRAB configuration for a given job.

NOTE: This is _not_ a complete (as in: submittable) CRAB
configuration. It is used to store the common settings for the
multicrab configuration.

NOTE: Only CERN CASTOR area (/castor/cern.ch/) is supported.

NOTE: According to CRAB, you `Must define exactly two of
total_number_of_events, events_per_job, or
number_of_jobs.'. For single-step harvesting we force one job,
for the rest we don't really care.

# BUG BUG BUG
# With the current version of CRAB (2.6.1), in which Daniele
# fixed the behaviour of no_block_boundary for me, one _has to
# specify_ the total_number_of_events and one single site in
# the se_white_list.
# BUG BUG BUG end

Definition at line 4233 of file cmsHarvester.py.

4233 
4234  def create_crab_config(self):
4235  """Create a CRAB configuration for a given job.
4236 
4237  NOTE: This is _not_ a complete (as in: submittable) CRAB
4238  configuration. It is used to store the common settings for the
4239  multicrab configuration.
4240 
4241  NOTE: Only CERN CASTOR area (/castor/cern.ch/) is supported.
4242 
4243  NOTE: According to CRAB, you `Must define exactly two of
4244  total_number_of_events, events_per_job, or
4245  number_of_jobs.'. For single-step harvesting we force one job,
4246  for the rest we don't really care.
4247 
4248  # BUG BUG BUG
4249  # With the current version of CRAB (2.6.1), in which Daniele
4250  # fixed the behaviour of no_block_boundary for me, one _has to
4251  # specify_ the total_number_of_events and one single site in
4252  # the se_white_list.
4253  # BUG BUG BUG end
4254 
4255  """
4256 
4257  tmp = []
4258 
4259  # This is the stuff we will need to fill in.
4260  castor_prefix = self.castor_prefix
4261 
4262  tmp.append(self.config_file_header())
4263  tmp.append("")
cmsHarvester.create_crab_config
def create_crab_config
Definition: cmsHarvester.py:4233
def cmsHarvester.create_es_prefer_snippet (   self,
  dataset_name 
) 
Build the es_prefer snippet for the reference histograms.

The building of the snippet is wrapped in some care-taking
code that figures out the name of the reference histogram set
and makes sure the corresponding tag exists.

Definition at line 4690 of file cmsHarvester.py.

References join().

4690 
4691  def create_es_prefer_snippet(self, dataset_name):
4692  """Build the es_prefer snippet for the reference histograms.
4693 
4694  The building of the snippet is wrapped in some care-taking
4695  code that figures out the name of the reference histogram set
4696  and makes sure the corresponding tag exists.
4697 
4698  """
4699 
4700  # Figure out the name of the reference histograms tag.
4701  # NOTE: The existence of these tags has already been checked.
4702  ref_hist_tag_name = self.ref_hist_mappings[dataset_name]
4703 
4704  connect_name = self.frontier_connection_name["refhists"]
4705  connect_name += self.db_account_name_cms_cond_dqm_summary()
4706  record_name = "DQMReferenceHistogramRootFileRcd"
4707 
4708  # Build up the code snippet.
4709  code_lines = []
4710  code_lines.append("from CondCore.DBCommon.CondDBSetup_cfi import *")
4711  code_lines.append("process.ref_hist_source = cms.ESSource(\"PoolDBESSource\", CondDBSetup,")
4712  code_lines.append(" connect = cms.string(\"%s\")," % connect_name)
4713  code_lines.append(" toGet = cms.VPSet(cms.PSet(record = cms.string(\"%s\")," % record_name)
4714  code_lines.append(" tag = cms.string(\"%s\"))," % ref_hist_tag_name)
4715  code_lines.append(" )")
4716  code_lines.append(" )")
4717  code_lines.append("process.es_prefer_ref_hist_source = cms.ESPrefer(\"PoolDBESSource\", \"ref_hist_source\")")
4718 
4719  snippet = "\n".join(code_lines)
4720 
4721  # End of create_es_prefer_snippet.
4722  return snippet
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
cmsHarvester.create_es_prefer_snippet
def create_es_prefer_snippet
Definition: cmsHarvester.py:4690
def cmsHarvester.create_harvesting_config (   self,
  dataset_name 
) 
Create the Python harvesting configuration for harvesting.

The basic configuration is created by
Configuration.PyReleaseValidation.ConfigBuilder. (This mimics
what cmsDriver.py does.) After that we add some specials
ourselves.

NOTE: On one hand it may not be nice to circumvent
cmsDriver.py, on the other hand cmsDriver.py does not really
do anything itself. All the real work is done by the
ConfigBuilder so there is not much risk that we miss out on
essential developments of cmsDriver in the future.

Definition at line 4725 of file cmsHarvester.py.

4725 
4726  def create_harvesting_config(self, dataset_name):
4727  """Create the Python harvesting configuration for harvesting.
4728 
4729  The basic configuration is created by
4730  Configuration.PyReleaseValidation.ConfigBuilder. (This mimics
4731  what cmsDriver.py does.) After that we add some specials
4732  ourselves.
4733 
4734  NOTE: On one hand it may not be nice to circumvent
4735  cmsDriver.py, on the other hand cmsDriver.py does not really
4736  do anything itself. All the real work is done by the
4737  ConfigBuilder so there is not much risk that we miss out on
4738  essential developments of cmsDriver in the future.
4739 
4740  """
4741 
4742  # Setup some options needed by the ConfigBuilder.
4743  config_options = defaultOptions
4744 
4745  # These are fixed for all kinds of harvesting jobs. Some of
4746  # them are not needed for the harvesting config, but to keep
4747  # the ConfigBuilder happy.
4748  config_options.name = "harvesting"
4749  config_options.scenario = "pp"
4750  config_options.number = 1
4751  config_options.arguments = self.ident_string()
4752  config_options.evt_type = config_options.name
4753  config_options.customisation_file = None
4754  config_options.filein = "dummy_value"
4755  config_options.filetype = "EDM"
4756  # This seems to be new in CMSSW 3.3.X, no clue what it does.
4757  config_options.gflash = "dummy_value"
4758  # This seems to be new in CMSSW 3.3.0.pre6, no clue what it
4759  # does.
#config_options.himix = "dummy_value"
cmsHarvester.create_harvesting_config
def create_harvesting_config
Definition: cmsHarvester.py:4725
def cmsHarvester.create_harvesting_config_file_name (   self,
  dataset_name 
)

Definition at line 4097 of file cmsHarvester.py.

Referenced by write_harvesting_config().

4097 
4098  def create_harvesting_config_file_name(self, dataset_name):
4099  "Generate the name to be used for the harvesting config file."
4100 
4101  file_name_base = "harvesting.py"
4102  dataset_name_escaped = self.escape_dataset_name(dataset_name)
4103  config_file_name = file_name_base.replace(".py",
4104  "_%s.py" % \
4105  dataset_name_escaped)
4106 
4107  # End of create_harvesting_config_file_name.
4108  return config_file_name
cmsHarvester.create_harvesting_config_file_name
def create_harvesting_config_file_name
Definition: cmsHarvester.py:4097
def cmsHarvester.create_harvesting_output_file_name (   self,
  dataset_name,
  run_number 
) 
Generate the name to be used for the harvesting output file.

This harvesting output file is the _final_ ROOT output file
containing the harvesting results. In case of two-step
harvesting there is an intermediate ME output file as well.

Definition at line 4169 of file cmsHarvester.py.

4169 
4170  def create_harvesting_output_file_name(self, dataset_name, run_number):
4171  """Generate the name to be used for the harvesting output file.
4172 
4173  This harvesting output file is the _final_ ROOT output file
4174  containing the harvesting results. In case of two-step
4175  harvesting there is an intermediate ME output file as well.
4176 
4177  """
4178 
4179  dataset_name_escaped = self.escape_dataset_name(dataset_name)
4180 
4181  # Hmmm, looking at the code for the DQMFileSaver this might
4182  # actually be the place where the first part of this file
4183  # naming scheme comes from.
4184  # NOTE: It looks like the `V0001' comes from the DQM
4185  # version. This is something that cannot be looked up from
4186  # here, so let's hope it does not change too often.
4187  output_file_name = "DQM_V0001_R%09d__%s.root" % \
4188  (run_number, dataset_name_escaped)
4189  if self.harvesting_mode.find("partial") > -1:
4190  # Only add the alarming piece to the file name if this is
4191  # a spread-out dataset.
4192  if self.datasets_information[dataset_name] \
4193  ["mirrored"][run_number] == False:
4194  output_file_name = output_file_name.replace(".root", \
4195  "_partial.root")
4196 
4197  # End of create_harvesting_output_file_name.
4198  return output_file_name
cmsHarvester.create_harvesting_output_file_name
def create_harvesting_output_file_name
Definition: cmsHarvester.py:4169
def cmsHarvester.create_me_extraction_config (   self,
  dataset_name 
) 

def create_harvesting_config_two_step(self, dataset_name):
    """Create the Python harvesting configuration for two-step
    harvesting.         """         # BUG BUG BUG
    config_contents = self.create_harvesting_config_single_step(dataset_name)

BUG BUG BUG end

End of create_harvesting_config_two_step.

return config_contents 

 

Definition at line 4951 of file cmsHarvester.py.



References create_output_file_name(), and join().


 4951 


 4952     def create_me_extraction_config(self, dataset_name):


 4953         """


 4954 


 4955         """


 4956 


 4957         # Big chunk of hard-coded Python. Not such a big deal since


 4958         # this does not do much and is not likely to break.


 4959         tmp = []


 4960         tmp.append(self.config_file_header())


 4961         tmp.append("")


 4962         tmp.append("import FWCore.ParameterSet.Config as cms")


 4963         tmp.append("")


 4964         tmp.append("process = cms.Process(\"ME2EDM\")")


 4965         tmp.append("")


 4966         tmp.append("# Import of standard configurations")


 4967         tmp.append("process.load(\"Configuration/EventContent/EventContent_cff\")")


 4968         tmp.append("")


 4969         tmp.append("# We don't really process any events, just keep this set to one to")


 4970         tmp.append("# make sure things work.")


 4971         tmp.append("process.maxEvents = cms.untracked.PSet(")


 4972         tmp.append("    input = cms.untracked.int32(1)")


 4973         tmp.append("    )")


 4974         tmp.append("")


 4975         tmp.append("process.options = cms.untracked.PSet(")


 4976         tmp.append("    Rethrow = cms.untracked.vstring(\"ProductNotFound\")")


 4977         tmp.append("    )")


 4978         tmp.append("")


 4979         tmp.append("process.source = cms.Source(\"PoolSource\",")


 4980         tmp.append("                            processingMode = \\")


 4981         tmp.append("                            cms.untracked.string(\"RunsAndLumis\"),")


 4982         tmp.append("                            fileNames = \\")


 4983         tmp.append("                            cms.untracked.vstring(\"no_file_specified\")")


 4984         tmp.append("                            )")


 4985         tmp.append("")


 4986         tmp.append("# Output definition: drop everything except for the monitoring.")


 4987         tmp.append("process.output = cms.OutputModule(")


 4988         tmp.append("    \"PoolOutputModule\",")


 4989         tmp.append("    outputCommands = \\")


 4990         tmp.append("    cms.untracked.vstring(\"drop *\", \\")


 4991         tmp.append("                          \"keep *_MEtoEDMConverter_*_*\"),")


 4992         output_file_name = self. \


 4993                            create_output_file_name(dataset_name)


 4994         tmp.append("    fileName = \\")


 4995         tmp.append("    cms.untracked.string(\"%s\")," % output_file_name)


 4996         tmp.append("    dataset = cms.untracked.PSet(")


 4997         tmp.append("    dataTier = cms.untracked.string(\"RECO\"),")


 4998         tmp.append("    filterName = cms.untracked.string(\"\")")


 4999         tmp.append("    )")


 5000         tmp.append("    )")


 5001         tmp.append("")


 5002         tmp.append("# Additional output definition")


 5003         tmp.append("process.out_step = cms.EndPath(process.output)")


 5004         tmp.append("")


 5005         tmp.append("# Schedule definition")


 5006         tmp.append("process.schedule = cms.Schedule(process.out_step)")


 5007         tmp.append("")


 5008 


 5009         config_contents = "\n".join(tmp)


 5010 


 5011         # End of create_me_extraction_config.


 5012         return config_contents




cmsHarvester.create_me_extraction_config
def create_me_extraction_config
Definition: cmsHarvester.py:4951


cmsHarvester.create_output_file_name
def create_output_file_name
Definition: cmsHarvester.py:4125


join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18











      

        

          def cmsHarvester.create_me_summary_config_file_name 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      





Definition at line 4111 of file cmsHarvester.py.



Referenced by write_me_extraction_config().


 4111 


 4112     def create_me_summary_config_file_name(self, dataset_name):


 4113         "Generate the name of the ME summary extraction config file."


 4114 


 4115         file_name_base = "me_extraction.py"


 4116         dataset_name_escaped = self.escape_dataset_name(dataset_name)


 4117         config_file_name = file_name_base.replace(".py",


 4118                                                   "_%s.py" % \


 4119                                                   dataset_name_escaped)


 4120 


 4121         # End of create_me_summary_config_file_name.


 4122         return config_file_name




cmsHarvester.create_me_summary_config_file_name
def create_me_summary_config_file_name
Definition: cmsHarvester.py:4111











      

        

          def cmsHarvester.create_me_summary_output_file_name 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      




Generate the name of the intermediate ME file name to be
used in two-step harvesting.
 

Definition at line 4201 of file cmsHarvester.py.


 4201 


 4202     def create_me_summary_output_file_name(self, dataset_name):


 4203         """Generate the name of the intermediate ME file name to be


 4204         used in two-step harvesting.


 4205 


 4206         """


 4207 


 4208         dataset_name_escaped = self.escape_dataset_name(dataset_name)


 4209         output_file_name = "me_summary_%s.root" % \


 4210                            dataset_name_escaped


 4211 


 4212         # End of create_me_summary_output_file_name.


 4213         return output_file_name




cmsHarvester.create_me_summary_output_file_name
def create_me_summary_output_file_name
Definition: cmsHarvester.py:4201











      

        

          def cmsHarvester.create_multicrab_block_name 
          (
           
          self, 
        

        

          
          
           
          dataset_name, 
        

        

          
          
           
          run_number, 
        

        

          
          
           
          index 
        

        

          
          )
          
        

      




Create the block name to use for this dataset/run number.

This is what appears in the brackets `[]' in multicrab.cfg. It
is used as the name of the job and to create output
directories.
 

Definition at line 4216 of file cmsHarvester.py.


 4216 


 4217     def create_multicrab_block_name(self, dataset_name, run_number, index):


 4218         """Create the block name to use for this dataset/run number.


 4219 


 4220         This is what appears in the brackets `[]' in multicrab.cfg. It


 4221         is used as the name of the job and to create output


 4222         directories.


 4223 


 4224         """


 4225 


 4226         dataset_name_escaped = self.escape_dataset_name(dataset_name)


 4227         block_name = "%s_%09d_%s" % (dataset_name_escaped, run_number, index)


 4228 


 4229         # End of create_multicrab_block_name.


 4230         return block_name




cmsHarvester.create_multicrab_block_name
def create_multicrab_block_name
Definition: cmsHarvester.py:4216











      

        

          def cmsHarvester.create_multicrab_config 
          (
           
          self)
          
        

      




Create a multicrab.cfg file for all samples.

This creates the contents for a multicrab.cfg file that uses
the crab.cfg file (generated elsewhere) for the basic settings
and contains blocks for each run of each dataset.

# BUG BUG BUG
# The fact that it's necessary to specify the se_white_list
# and the total_number_of_events is due to our use of CRAB
# version 2.6.1. This should no longer be necessary in the
# future.
# BUG BUG BUG end
 

Definition at line 4313 of file cmsHarvester.py.


 4313 


 4314     def create_multicrab_config(self):


 4315         """Create a multicrab.cfg file for all samples.


 4316 


 4317         This creates the contents for a multicrab.cfg file that uses


 4318         the crab.cfg file (generated elsewhere) for the basic settings


 4319         and contains blocks for each run of each dataset.


 4320 


 4321         # BUG BUG BUG


 4322         # The fact that it's necessary to specify the se_white_list


 4323         # and the total_number_of_events is due to our use of CRAB


 4324         # version 2.6.1. This should no longer be necessary in the


 4325         # future.


 4326         # BUG BUG BUG end


 4327 


 4328         """




cmsHarvester.create_multicrab_config
def create_multicrab_config
Definition: cmsHarvester.py:4313











      

        

          def cmsHarvester.create_output_file_name 
          (
           
          self, 
        

        

          
          
           
          dataset_name, 
        

        

          
          
           
          run_number = None 
        

        

          
          )
          
        

      




Create the name of the output file name to be used.

This is the name of the output file of the `first step'. In
the case of single-step harvesting this is already the final
harvesting output ROOT file. In the case of two-step
harvesting it is the name of the intermediary ME summary
file.
 

Definition at line 4125 of file cmsHarvester.py.



Referenced by create_me_extraction_config().


 4125 


 4126     def create_output_file_name(self, dataset_name, run_number=None):


 4127         """Create the name of the output file name to be used.


 4128 


 4129         This is the name of the output file of the `first step'. In


 4130         the case of single-step harvesting this is already the final


 4131         harvesting output ROOT file. In the case of two-step


 4132         harvesting it is the name of the intermediary ME summary


 4133         file.


 4134 


 4135         """


 4136 


 4137         # BUG BUG BUG


 4138         # This method has become a bit of a mess. Originally it was


 4139         # nice to have one entry point for both single- and two-step


 4140         # output file names. However, now the former needs the run


 4141         # number, while the latter does not even know about run


 4142         # numbers. This should be fixed up a bit.


 4143         # BUG BUG BUG end


 4144 


 4145         if self.harvesting_mode == "single-step":


 4146             # DEBUG DEBUG DEBUG


 4147             assert not run_number is None


 4148             # DEBUG DEBUG DEBUG end


 4149             output_file_name = self.create_harvesting_output_file_name(dataset_name, run_number)


 4150         elif self.harvesting_mode == "single-step-allow-partial":


 4151             # DEBUG DEBUG DEBUG


 4152             assert not run_number is None


 4153             # DEBUG DEBUG DEBUG end


 4154             output_file_name = self.create_harvesting_output_file_name(dataset_name, run_number)


 4155         elif self.harvesting_mode == "two-step":


 4156             # DEBUG DEBUG DEBUG


 4157             assert run_number is None


 4158             # DEBUG DEBUG DEBUG end


 4159             output_file_name = self.create_me_summary_output_file_name(dataset_name)


 4160         else:


 4161             # This should not be possible, but hey...


 4162             assert False, "ERROR Unknown harvesting mode `%s'" % \


 4163                    self.harvesting_mode


 4164 


 4165         # End of create_harvesting_output_file_name.


 4166         return output_file_name




cmsHarvester.create_output_file_name
def create_output_file_name
Definition: cmsHarvester.py:4125











      

        

          def cmsHarvester.dbs_check_dataset_spread 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      





def dbs_resolve_dataset_number_of_sites(self, dataset_name): """Ask DBS across how many sites this dataset has been spread out. 


     This is especially useful to check that we do not submit a job
    supposed to run on a complete sample that is not contained at
    a single site.         """         # DEBUG DEBUG DEBUG

 
If we get here DBS should have been set up already.


assert not self.dbs_api is None 


DEBUG DEBUG DEBUG end


api = self.dbs_api dbs_query = "find count(site) where dataset = %s " \ "and dataset.status = VALID" % \ dataset_name try: api_result = api.executeQuery(dbs_query) except DbsApiException: raise Error("ERROR: Could not execute DBS query") try: num_sites = [] class Handler(xml.sax.handler.ContentHandler): def startElement(self, name, attrs): if name == "result": num_sites.append(str(attrs["COUNT_STORAGEELEMENT"])) xml.sax.parseString(api_result, Handler()) except SAXParseException: raise Error("ERROR: Could not parse DBS server output") # DEBUG DEBUG DEBUG assert len(num_sites) == 1 


DEBUG DEBUG DEBUG end


num_sites = int(num_sites[0]) # End of dbs_resolve_dataset_number_of_sites. return num_sites def dbs_check_dataset_spread(self, dataset_name): """Figure out across how many sites this dataset is spread. NOTE: This is something we need to figure out per run, since we want to submit harvesting jobs per run. Basically three things can happen with a given dataset:



assert not self.dbs_api is None 


DEBUG DEBUG DEBUG end


api = self.dbs_api dbs_query = "find run, run.numevents, site, file.count " \ "where dataset = %s " \ "and dataset.status = VALID" % \ dataset_name try: api_result = api.executeQuery(dbs_query) except DbsApiException: msg = "ERROR: Could not execute DBS query" self.logger.fatal(msg) raise Error(msg) # Index things by run number. No cross-check is done to make 


sure we get results for each and every run in the


dataset. I'm not sure this would make sense since we'd be


cross-checking DBS info with DBS info anyway. Note that we


use the file count per site to see if we're dealing with an


incomplete vs. a mirrored dataset.


sample_info = {} try: class Handler(xml.sax.handler.ContentHandler): def startElement(self, name, attrs): if name == "result": run_number = int(attrs["RUNS_RUNNUMBER"]) site_name = str(attrs["STORAGEELEMENT_SENAME"]) file_count = int(attrs["COUNT_FILES"]) 


BUG BUG BUG


Doh! For some reason DBS never returns any other


event count than zero.


event_count = int(attrs["RUNS_NUMBEROFEVENTS"]) 


BUG BUG BUG end


info = (site_name, file_count, event_count) try: sample_info[run_number].append(info) except KeyError: sample_info[run_number] = [info] xml.sax.parseString(api_result, Handler()) except SAXParseException: msg = "ERROR: Could not parse DBS server output" self.logger.fatal(msg) raise Error(msg) # Now translate this into a slightly more usable mapping. sites = {} for (run_number, site_info) in sample_info.iteritems(): 


Quick-n-dirty trick to see if all file counts are the


same.


unique_file_counts = set([i[1] for i in site_info]) if len(unique_file_counts) == 1: 


Okay, so this must be a mirrored dataset.


We have to pick one but we have to be careful. We


cannot submit to things like a T0, a T1, or CAF.


site_names = [self.pick_a_site([i[0] for i in site_info])] nevents = [site_info[0][2]] else: 


Looks like this is a spread-out sample.


site_names = [i[0] for i in site_info] nevents = [i[2] for i in site_info] sites[run_number] = zip(site_names, nevents) self.logger.debug("Sample `%s' spread is:" % dataset_name) run_numbers = sites.keys() run_numbers.sort() for run_number in run_numbers: self.logger.debug("  run # %6d: %d sites (%s)" % \ (run_number, len(sites[run_number]), ", ".join([i[0] for i in sites[run_number]]))) # End of dbs_check_dataset_spread. return sites # DEBUG DEBUG DEBUG 


Just kept for debugging now.


def dbs_check_dataset_spread_old(self, dataset_name): """Figure out across how many sites this dataset is spread. NOTE: This is something we need to figure out per run, since we want to submit harvesting jobs per run. Basically three things can happen with a given dataset:



assert not self.dbs_api is None 


DEBUG DEBUG DEBUG end


api = self.dbs_api dbs_query = "find run, run.numevents, site, file.count " \ "where dataset = %s " \ "and dataset.status = VALID" % \ dataset_name try: api_result = api.executeQuery(dbs_query) except DbsApiException: msg = "ERROR: Could not execute DBS query" self.logger.fatal(msg) raise Error(msg) # Index things by run number. No cross-check is done to make 


sure we get results for each and every run in the


dataset. I'm not sure this would make sense since we'd be


cross-checking DBS info with DBS info anyway. Note that we


use the file count per site to see if we're dealing with an


incomplete vs. a mirrored dataset.


sample_info = {} try: class Handler(xml.sax.handler.ContentHandler): def startElement(self, name, attrs): if name == "result": run_number = int(attrs["RUNS_RUNNUMBER"]) site_name = str(attrs["STORAGEELEMENT_SENAME"]) file_count = int(attrs["COUNT_FILES"]) 


BUG BUG BUG


Doh! For some reason DBS never returns any other


event count than zero.


event_count = int(attrs["RUNS_NUMBEROFEVENTS"]) 


BUG BUG BUG end


info = (site_name, file_count, event_count) try: sample_info[run_number].append(info) except KeyError: sample_info[run_number] = [info] xml.sax.parseString(api_result, Handler()) except SAXParseException: msg = "ERROR: Could not parse DBS server output" self.logger.fatal(msg) raise Error(msg) # Now translate this into a slightly more usable mapping. sites = {} for (run_number, site_info) in sample_info.iteritems(): 


Quick-n-dirty trick to see if all file counts are the


same.


unique_file_counts = set([i[1] for i in site_info]) if len(unique_file_counts) == 1: 


Okay, so this must be a mirrored dataset.


We have to pick one but we have to be careful. We


cannot submit to things like a T0, a T1, or CAF.


site_names = [self.pick_a_site([i[0] for i in site_info])] nevents = [site_info[0][2]] else: 


Looks like this is a spread-out sample.


site_names = [i[0] for i in site_info] nevents = [i[2] for i in site_info] sites[run_number] = zip(site_names, nevents) self.logger.debug("Sample `%s' spread is:" % dataset_name) run_numbers = sites.keys() run_numbers.sort() for run_number in run_numbers: self.logger.debug("  run # %6d: %d site(s) (%s)" % \ (run_number, len(sites[run_number]), ", ".join([i[0] for i in sites[run_number]]))) # End of dbs_check_dataset_spread_old. return sites 


DEBUG DEBUG DEBUG end


Figure out the number of events in each run of this dataset.

This is a more efficient way of doing this than calling
dbs_resolve_number_of_events for each run.
 

Definition at line 3076 of file cmsHarvester.py.


 3076 


 3077     def dbs_check_dataset_spread(self, dataset_name):


 3078         """Figure out the number of events in each run of this dataset.


 3079 


 3080         This is a more efficient way of doing this than calling


 3081         dbs_resolve_number_of_events for each run.


 3082 


 3083         """


 3084 


 3085         self.logger.debug("Checking spread of dataset `%s'" % dataset_name)


 3086 


 3087         # DEBUG DEBUG DEBUG


 3088         # If we get here DBS should have been set up already.


 3089         assert not self.dbs_api is None


 3090         # DEBUG DEBUG DEBUG end


 3091 


 3092         api = self.dbs_api


 3093         dbs_query = "find run.number, site, file.name, file.numevents " \


 3094                     "where dataset = %s " \


 3095                     "and dataset.status = VALID" % \


 3096                     dataset_name


 3097         try:


 3098             api_result = api.executeQuery(dbs_query)


 3099         except DBSAPI.dbsApiException.DbsApiException:


 3100             msg = "ERROR: Could not execute DBS query"


 3101             self.logger.fatal(msg)


 3102             raise Error(msg)


 3103 


 3104         handler = DBSXMLHandler(["run.number", "site", "file.name", "file.numevents"])


 3105         parser = xml.sax.make_parser()


 3106         parser.setContentHandler(handler)


 3107 


 3108         try:

            # OBSOLETE OBSOLETE OBSOLETE


cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.DBSXMLHandler
Helper class: DBSXMLHandler. 
Definition: cmsHarvester.py:184


cmsHarvester.dbs_check_dataset_spread
def dbs_check_dataset_spread
def dbs_resolve_dataset_number_of_sites(self, dataset_name): &quot;&quot;"Ask DBS across how many sites this da...
Definition: cmsHarvester.py:3076











      

        

          def cmsHarvester.dbs_resolve_cmssw_version 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      




Ask DBS for the CMSSW version used to create this dataset.
 

Definition at line 2474 of file cmsHarvester.py.


 2474 


 2475     def dbs_resolve_cmssw_version(self, dataset_name):


 2476         """Ask DBS for the CMSSW version used to create this dataset.


 2477 


 2478         """


 2479 


 2480         # DEBUG DEBUG DEBUG


 2481         # If we get here DBS should have been set up already.


 2482         assert not self.dbs_api is None


 2483         # DEBUG DEBUG DEBUG end


 2484 


 2485         api = self.dbs_api


 2486         dbs_query = "find algo.version where dataset = %s " \


 2487                     "and dataset.status = VALID" % \


 2488                     dataset_name


 2489         try:


 2490             api_result = api.executeQuery(dbs_query)


 2491         except DBSAPI.dbsApiException.DbsApiException:


 2492             msg = "ERROR: Could not execute DBS query"


 2493             self.logger.fatal(msg)


 2494             raise Error(msg)


 2495 


 2496         handler = DBSXMLHandler(["algo.version"])


 2497         parser = xml.sax.make_parser()


 2498         parser.setContentHandler(handler)


 2499 


 2500         try:


 2501             xml.sax.parseString(api_result, handler)


 2502         except SAXParseException:


 2503             msg = "ERROR: Could not parse DBS server output"


 2504             self.logger.fatal(msg)


 2505             raise Error(msg)


 2506 


 2507         # DEBUG DEBUG DEBUG


 2508         assert(handler.check_results_validity()), "ERROR The DBSXMLHandler screwed something up!"


 2509         # DEBUG DEBUG DEBUG end


 2510 


 2511         cmssw_version = handler.results.values()[0]


 2512 


 2513         # DEBUG DEBUG DEBUG


 2514         assert len(cmssw_version) == 1


 2515         # DEBUG DEBUG DEBUG end


 2516 


 2517         cmssw_version = cmssw_version[0]


 2518 


 2519         # End of dbs_resolve_cmssw_version.


 2520         return cmssw_version




cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.DBSXMLHandler
Helper class: DBSXMLHandler. 
Definition: cmsHarvester.py:184


cmsHarvester.dbs_resolve_cmssw_version
def dbs_resolve_cmssw_version
Definition: cmsHarvester.py:2474











      

        

          def cmsHarvester.dbs_resolve_dataset_name 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      




Use DBS to resolve a wildcarded dataset name.
 

Definition at line 2418 of file cmsHarvester.py.



Referenced by build_dataset_list().


 2418 


 2419     def dbs_resolve_dataset_name(self, dataset_name):


 2420         """Use DBS to resolve a wildcarded dataset name.


 2421 


 2422         """


 2423 


 2424         # DEBUG DEBUG DEBUG


 2425         # If we get here DBS should have been set up already.


 2426         assert not self.dbs_api is None


 2427         # DEBUG DEBUG DEBUG end


 2428 


 2429         # Some minor checking to make sure that whatever we've been


 2430         # given as dataset name actually sounds like a dataset name.


 2431         if not (dataset_name.startswith("/") and \


 2432                 dataset_name.endswith("RECO")):


 2433             self.logger.warning("Dataset name `%s' does not sound " \


 2434                                  "like a valid dataset name!" % \


 2435                                 dataset_name)


 2436 


 2437         #----------


 2438 


 2439         api = self.dbs_api


 2440         dbs_query = "find dataset where dataset like %s " \


 2441                     "and dataset.status = VALID" % \


 2442                     dataset_name


 2443         try:


 2444             api_result = api.executeQuery(dbs_query)


 2445         except DBSAPI.dbsApiException.DbsApiException:


 2446             msg = "ERROR: Could not execute DBS query"


 2447             self.logger.fatal(msg)


 2448             raise Error(msg)


 2449 


 2450         # Setup parsing.


 2451         handler = DBSXMLHandler(["dataset"])


 2452         parser = xml.sax.make_parser()


 2453         parser.setContentHandler(handler)


 2454 


 2455         # Parse.


 2456         try:


 2457             xml.sax.parseString(api_result, handler)


 2458         except SAXParseException:


 2459             msg = "ERROR: Could not parse DBS server output"


 2460             self.logger.fatal(msg)


 2461             raise Error(msg)


 2462 


 2463         # DEBUG DEBUG DEBUG


 2464         assert(handler.check_results_validity()), "ERROR The DBSXMLHandler screwed something up!"


 2465         # DEBUG DEBUG DEBUG end


 2466 


 2467         # Extract the results.


 2468         datasets = handler.results.values()[0]


 2469 


 2470         # End of dbs_resolve_dataset_name.


 2471         return datasets




cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.DBSXMLHandler
Helper class: DBSXMLHandler. 
Definition: cmsHarvester.py:184


cmsHarvester.dbs_resolve_dataset_name
def dbs_resolve_dataset_name
Definition: cmsHarvester.py:2418











      

        

          def cmsHarvester.dbs_resolve_datatype 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      




Ask DBS for the the data type (data or mc) of a given
dataset.
 

Definition at line 2682 of file cmsHarvester.py.


 2682 


 2683     def dbs_resolve_datatype(self, dataset_name):


 2684         """Ask DBS for the the data type (data or mc) of a given


 2685         dataset.


 2686 


 2687         """


 2688 


 2689         # DEBUG DEBUG DEBUG


 2690         # If we get here DBS should have been set up already.


 2691         assert not self.dbs_api is None


 2692         # DEBUG DEBUG DEBUG end


 2693 


 2694         api = self.dbs_api


 2695         dbs_query = "find datatype.type where dataset = %s " \


 2696                     "and dataset.status = VALID" % \


 2697                     dataset_name


 2698         try:


 2699             api_result = api.executeQuery(dbs_query)


 2700         except DBSAPI.dbsApiException.DbsApiException:


 2701             msg = "ERROR: Could not execute DBS query"


 2702             self.logger.fatal(msg)


 2703             raise Error(msg)


 2704 


 2705         handler = DBSXMLHandler(["datatype.type"])


 2706         parser = xml.sax.make_parser()


 2707         parser.setContentHandler(handler)


 2708 


 2709         try:


 2710             xml.sax.parseString(api_result, handler)


 2711         except SAXParseException:


 2712             msg = "ERROR: Could not parse DBS server output"


 2713             self.logger.fatal(msg)


 2714             raise Error(msg)


 2715 


 2716         # DEBUG DEBUG DEBUG


 2717         assert(handler.check_results_validity()), "ERROR The DBSXMLHandler screwed something up!"


 2718         # DEBUG DEBUG DEBUG end


 2719 


 2720         datatype = handler.results.values()[0]


 2721 


 2722         # DEBUG DEBUG DEBUG


 2723         assert len(datatype) == 1


 2724         # DEBUG DEBUG DEBUG end


 2725 


 2726         datatype = datatype[0]


 2727 


 2728         # End of dbs_resolve_datatype.


 2729         return datatype




cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.DBSXMLHandler
Helper class: DBSXMLHandler. 
Definition: cmsHarvester.py:184


cmsHarvester.dbs_resolve_datatype
def dbs_resolve_datatype
Definition: cmsHarvester.py:2682











      

        

          def cmsHarvester.dbs_resolve_globaltag 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      




Ask DBS for the globaltag corresponding to a given dataset.

# BUG BUG BUG
# This does not seem to work for data datasets? E.g. for
# /Cosmics/Commissioning08_CRAFT0831X_V1_311_ReReco_FromSuperPointing_v1/RAW-RECO
# Probaly due to the fact that the GlobalTag changed during
# datataking...
BUG BUG BUG end
 

Definition at line 2626 of file cmsHarvester.py.


 2626 


 2627     def dbs_resolve_globaltag(self, dataset_name):


 2628         """Ask DBS for the globaltag corresponding to a given dataset.


 2629 


 2630         # BUG BUG BUG


 2631         # This does not seem to work for data datasets? E.g. for


 2632         # /Cosmics/Commissioning08_CRAFT0831X_V1_311_ReReco_FromSuperPointing_v1/RAW-RECO


 2633         # Probaly due to the fact that the GlobalTag changed during


 2634         # datataking...


 2635         BUG BUG BUG end


 2636 


 2637         """


 2638 


 2639         # DEBUG DEBUG DEBUG


 2640         # If we get here DBS should have been set up already.


 2641         assert not self.dbs_api is None


 2642         # DEBUG DEBUG DEBUG end


 2643 


 2644         api = self.dbs_api


 2645         dbs_query = "find dataset.tag where dataset = %s " \


 2646                     "and dataset.status = VALID" % \


 2647                     dataset_name


 2648         try:


 2649             api_result = api.executeQuery(dbs_query)


 2650         except DBSAPI.dbsApiException.DbsApiException:


 2651             msg = "ERROR: Could not execute DBS query"


 2652             self.logger.fatal(msg)


 2653             raise Error(msg)


 2654 


 2655         handler = DBSXMLHandler(["dataset.tag"])


 2656         parser = xml.sax.make_parser()


 2657         parser.setContentHandler(parser)


 2658 


 2659         try:


 2660             xml.sax.parseString(api_result, handler)


 2661         except SAXParseException:


 2662             msg = "ERROR: Could not parse DBS server output"


 2663             self.logger.fatal(msg)


 2664             raise Error(msg)


 2665 


 2666         # DEBUG DEBUG DEBUG


 2667         assert(handler.check_results_validity()), "ERROR The DBSXMLHandler screwed something up!"


 2668         # DEBUG DEBUG DEBUG end


 2669 


 2670         globaltag = handler.results.values()[0]


 2671 


 2672         # DEBUG DEBUG DEBUG


 2673         assert len(globaltag) == 1


 2674         # DEBUG DEBUG DEBUG end


 2675 


 2676         globaltag = globaltag[0]


 2677 


 2678         # End of dbs_resolve_globaltag.


 2679         return globaltag




cmsHarvester.dbs_resolve_globaltag
def dbs_resolve_globaltag
Definition: cmsHarvester.py:2626


cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.DBSXMLHandler
Helper class: DBSXMLHandler. 
Definition: cmsHarvester.py:184











      

        

          def cmsHarvester.dbs_resolve_number_of_events 
          (
           
          self, 
        

        

          
          
           
          dataset_name, 
        

        

          
          
           
          run_number = None 
        

        

          
          )
          
        

      




Determine the number of events in a given dataset (and run).

Ask DBS for the number of events in a dataset. If a run number
is specified the number of events returned is that in that run
of that dataset. If problems occur we throw an exception.

# BUG BUG BUG
# Since DBS does not return the number of events correctly,
# neither for runs nor for whole datasets, we have to work
# around that a bit...
# BUG BUG BUG end
 

Definition at line 2735 of file cmsHarvester.py.


 2735 


 2736     def dbs_resolve_number_of_events(self, dataset_name, run_number=None):


 2737         """Determine the number of events in a given dataset (and run).


 2738 


 2739         Ask DBS for the number of events in a dataset. If a run number


 2740         is specified the number of events returned is that in that run


 2741         of that dataset. If problems occur we throw an exception.


 2742 


 2743         # BUG BUG BUG


 2744         # Since DBS does not return the number of events correctly,


 2745         # neither for runs nor for whole datasets, we have to work


 2746         # around that a bit...


 2747         # BUG BUG BUG end


 2748 


 2749         """


 2750 


 2751         # DEBUG DEBUG DEBUG


 2752         # If we get here DBS should have been set up already.


 2753         assert not self.dbs_api is None


 2754         # DEBUG DEBUG DEBUG end


 2755 


 2756         api = self.dbs_api


 2757         dbs_query = "find file.name, file.numevents where dataset = %s " \


 2758                     "and dataset.status = VALID" % \


 2759                     dataset_name


 2760         if not run_number is None:


 2761             dbs_query = dbq_query + (" and run = %d" % run_number)


 2762         try:


 2763             api_result = api.executeQuery(dbs_query)


 2764         except DBSAPI.dbsApiException.DbsApiException:


 2765             msg = "ERROR: Could not execute DBS query"


 2766             self.logger.fatal(msg)


 2767             raise Error(msg)


 2768 


 2769         handler = DBSXMLHandler(["file.name", "file.numevents"])


 2770         parser = xml.sax.make_parser()


 2771         parser.setContentHandler(handler)


 2772 


 2773         try:


 2774             xml.sax.parseString(api_result, handler)


 2775         except SAXParseException:


 2776             msg = "ERROR: Could not parse DBS server output"


 2777             self.logger.fatal(msg)


 2778             raise Error(msg)


 2779 


 2780         # DEBUG DEBUG DEBUG


 2781         assert(handler.check_results_validity()), "ERROR The DBSXMLHandler screwed something up!"


 2782         # DEBUG DEBUG DEBUG end


 2783 


 2784         num_events = sum(handler.results["file.numevents"])


 2785 


 2786         # End of dbs_resolve_number_of_events.


 2787         return num_events




cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.DBSXMLHandler
Helper class: DBSXMLHandler. 
Definition: cmsHarvester.py:184


cmsHarvester.dbs_resolve_number_of_events
def dbs_resolve_number_of_events
Definition: cmsHarvester.py:2735











      

        

          def cmsHarvester.dbs_resolve_runs 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      





def dbs_resolve_dataset_number_of_events(self, dataset_name): """Ask DBS across how many events this dataset has been spread out. 


     This is especially useful to check that we do not submit a job
    supposed to run on a complete sample that is not contained at
    a single site.         """         # DEBUG DEBUG DEBUG

 
If we get here DBS should have been set up already.


assert not self.dbs_api is None 


DEBUG DEBUG DEBUG end


api = self.dbs_api dbs_query = "find count(site) where dataset = %s " \ "and dataset.status = VALID" % \ dataset_name try: api_result = api.executeQuery(dbs_query) except DbsApiException: raise Error("ERROR: Could not execute DBS query") try: num_events = [] class Handler(xml.sax.handler.ContentHandler): def startElement(self, name, attrs): if name == "result": num_events.append(str(attrs["COUNT_STORAGEELEMENT"])) xml.sax.parseString(api_result, Handler()) except SAXParseException: raise Error("ERROR: Could not parse DBS server output") # DEBUG DEBUG DEBUG assert len(num_events) == 1 


DEBUG DEBUG DEBUG end


num_events = int(num_events[0]) # End of dbs_resolve_dataset_number_of_events. return num_events 


Ask DBS for the list of runs in a given dataset.

# NOTE: This does not (yet?) skip/remove empty runs. There is
# a bug in the DBS entry run.numevents (i.e. it always returns
# zero) which should be fixed in the `next DBS release'.
# See also:
#   https://savannah.cern.ch/bugs/?53452
#   https://savannah.cern.ch/bugs/?53711
 

Definition at line 2568 of file cmsHarvester.py.


 2568 


 2569     def dbs_resolve_runs(self, dataset_name):


 2570         """Ask DBS for the list of runs in a given dataset.


 2571 


 2572         # NOTE: This does not (yet?) skip/remove empty runs. There is


 2573         # a bug in the DBS entry run.numevents (i.e. it always returns


 2574         # zero) which should be fixed in the `next DBS release'.


 2575         # See also:


 2576         #   https://savannah.cern.ch/bugs/?53452


 2577         #   https://savannah.cern.ch/bugs/?53711


 2578 


 2579         """


 2580 


 2581         # TODO TODO TODO


 2582         # We should remove empty runs as soon as the above mentioned


 2583         # bug is fixed.


 2584         # TODO TODO TODO end


 2585 


 2586         # DEBUG DEBUG DEBUG


 2587         # If we get here DBS should have been set up already.


 2588         assert not self.dbs_api is None


 2589         # DEBUG DEBUG DEBUG end


 2590 


 2591         api = self.dbs_api


 2592         dbs_query = "find run where dataset = %s " \


 2593                     "and dataset.status = VALID" % \


 2594                     dataset_name


 2595         try:


 2596             api_result = api.executeQuery(dbs_query)


 2597         except DBSAPI.dbsApiException.DbsApiException:


 2598             msg = "ERROR: Could not execute DBS query"


 2599             self.logger.fatal(msg)


 2600             raise Error(msg)


 2601 


 2602         handler = DBSXMLHandler(["run"])


 2603         parser = xml.sax.make_parser()


 2604         parser.setContentHandler(handler)


 2605 


 2606         try:


 2607             xml.sax.parseString(api_result, handler)


 2608         except SAXParseException:


 2609             msg = "ERROR: Could not parse DBS server output"


 2610             self.logger.fatal(msg)


 2611             raise Error(msg)


 2612 


 2613         # DEBUG DEBUG DEBUG


 2614         assert(handler.check_results_validity()), "ERROR The DBSXMLHandler screwed something up!"


 2615         # DEBUG DEBUG DEBUG end


 2616 


 2617         runs = handler.results.values()[0]


 2618         # Turn strings into integers.


 2619         runs = [int(i) for i in runs]


 2620         runs.sort()


 2621 


 2622         # End of dbs_resolve_runs.


 2623         return runs




cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.DBSXMLHandler
Helper class: DBSXMLHandler. 
Definition: cmsHarvester.py:184


cmsHarvester.dbs_resolve_runs
def dbs_resolve_runs
def dbs_resolve_dataset_number_of_events(self, dataset_name): &quot;&quot;"Ask DBS across how many events this ...
Definition: cmsHarvester.py:2568











      

        

          def cmsHarvester.escape_dataset_name 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      




Escape a DBS dataset name.

Escape a DBS dataset name such that it does not cause trouble
with the file system. This means turning each `/' into `__',
except for the first one which is just removed.
 

Definition at line 4046 of file cmsHarvester.py.


 4046 


 4047     def escape_dataset_name(self, dataset_name):


 4048         """Escape a DBS dataset name.


 4049 


 4050         Escape a DBS dataset name such that it does not cause trouble


 4051         with the file system. This means turning each `/' into `__',


 4052         except for the first one which is just removed.


 4053 


 4054         """


 4055 


 4056         escaped_dataset_name = dataset_name


 4057         escaped_dataset_name = escaped_dataset_name.strip("/")


 4058         escaped_dataset_name = escaped_dataset_name.replace("/", "__")


 4059 


 4060         return escaped_dataset_name




cmsHarvester.escape_dataset_name
def escape_dataset_name
Definition: cmsHarvester.py:4046











      

        

          def cmsHarvester.load_ref_hist_mappings 
          (
           
          self)
          
        

      




Load the reference histogram mappings from file.

The dataset name to reference histogram name mappings are read
from a text file specified in self.ref_hist_mappings_file_name.
 

Definition at line 5206 of file cmsHarvester.py.



References mergeVDriftHistosByStation.file, and bookConverter.max.


 5206 


 5207     def load_ref_hist_mappings(self):


 5208         """Load the reference histogram mappings from file.


 5209 


 5210         The dataset name to reference histogram name mappings are read


 5211         from a text file specified in self.ref_hist_mappings_file_name.


 5212 


 5213         """


 5214 


 5215         # DEBUG DEBUG DEBUG


 5216         assert len(self.ref_hist_mappings) < 1, \


 5217                "ERROR Should not be RE-loading " \


 5218                "reference histogram mappings!"


 5219         # DEBUG DEBUG DEBUG end


 5220 


 5221         self.logger.info("Loading reference histogram mappings " \


 5222                          "from file `%s'" % \


 5223                          self.ref_hist_mappings_file_name)


 5224 


 5225         mappings_lines = None


 5226         try:


 5227             mappings_file = file(self.ref_hist_mappings_file_name, "r")


 5228             mappings_lines = mappings_file.readlines()


 5229             mappings_file.close()


 5230         except IOError:


 5231             msg = "ERROR: Could not open reference histogram mapping "\


 5232                   "file `%s'" % self.ref_hist_mappings_file_name


 5233             self.logger.fatal(msg)


 5234             raise Error(msg)


 5235 


 5236         ##########


 5237 


 5238         # The format we expect is: two white-space separated pieces


 5239         # per line. The first the dataset name for which the reference


 5240         # should be used, the second one the name of the reference


 5241         # histogram in the database.


 5242 


 5243         for mapping in mappings_lines:


 5244             # Skip comment lines.


 5245             if not mapping.startswith("#"):


 5246                 mapping = mapping.strip()


 5247                 if len(mapping) > 0:


 5248                     mapping_pieces = mapping.split()


 5249                     if len(mapping_pieces) != 2:


 5250                         msg = "ERROR: The reference histogram mapping " \


 5251                               "file contains a line I don't " \


 5252                               "understand:\n  %s" % mapping


 5253                         self.logger.fatal(msg)


 5254                         raise Error(msg)


 5255                     dataset_name = mapping_pieces[0].strip()


 5256                     ref_hist_name = mapping_pieces[1].strip()


 5257                     # We don't want people to accidentally specify


 5258                     # multiple mappings for the same dataset. Just


 5259                     # don't accept those cases.


 5260                     if self.ref_hist_mappings.has_key(dataset_name):


 5261                         msg = "ERROR: The reference histogram mapping " \


 5262                               "file contains multiple mappings for " \


 5263                               "dataset `%s'."


 5264                         self.logger.fatal(msg)


 5265                         raise Error(msg)


 5266 


 5267                     # All is well that ends well.


 5268                     self.ref_hist_mappings[dataset_name] = ref_hist_name


 5269 


 5270         ##########


 5271 


 5272         self.logger.info("  Successfully loaded %d mapping(s)" % \


 5273                          len(self.ref_hist_mappings))


 5274         max_len = max([len(i) for i in self.ref_hist_mappings.keys()])


 5275         for (map_from, map_to) in self.ref_hist_mappings.iteritems():


 5276             self.logger.info("    %-*s -> %s" % \


 5277                               (max_len, map_from, map_to))


 5278 


 5279         # End of load_ref_hist_mappings.




cmsHarvester.load_ref_hist_mappings
def load_ref_hist_mappings
Definition: cmsHarvester.py:5206



cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139












      

        

          def cmsHarvester.option_handler_caf_access 
          (
           
          self, 
        

        

          
          
           
          option, 
        

        

          
          
           
          opt_str, 
        

        

          
          
           
          value, 
        

        

          
          
           
          parser 
        

        

          
          )
          
        

      




Set the self.caf_access flag to try and create jobs that
run on the CAF.
 

Definition at line 1100 of file cmsHarvester.py.


 1100 


 1101     def option_handler_caf_access(self, option, opt_str, value, parser):


 1102         """Set the self.caf_access flag to try and create jobs that


 1103         run on the CAF.


 1104 


 1105         """


 1106         self.caf_access = True


 1107 


 1108         self.logger.warning("Running in `caf_access' mode. " \


 1109                             "Will try to create jobs that run " \


 1110                             "on CAF but no" \


 1111                             "further promises...")


 1112 


 1113         # End of option_handler_caf_access.

        


cmsHarvester.option_handler_caf_access
def option_handler_caf_access
Definition: cmsHarvester.py:1100











      

        

          def cmsHarvester.option_handler_castor_dir 
          (
           
          self, 
        

        

          
          
           
          option, 
        

        

          
          
           
          opt_str, 
        

        

          
          
           
          value, 
        

        

          
          
           
          parser 
        

        

          
          )
          
        

      





def option_handler_dataset_name(self, option, opt_str, value, parser): """Specify the name(s) of the dataset(s) to be processed. 


     It is checked to make sure that no dataset name or listfile
    names are given yet. If all is well (i.e. we still have a
    clean slate) the dataset name is stored for later use,
    otherwise a Usage exception is raised.         """         if not self.input_method is None:
        if self.input_method == "dataset":
            raise Usage("Please only feed me one dataset specification")
        elif self.input_method == "listfile":
            raise Usage("Cannot specify both dataset and input list file")
        else:
            assert False, "Unknown input method `%s'" % self.input_method
    self.input_method = "dataset"
    self.input_name = value
    self.logger.info("Input method used: %s" % self.input_method)         # End of option_handler_dataset_name.     ##########     def option_handler_listfile_name(self, option, opt_str, value, parser):
    """Specify the input list file containing datasets to be processed.         It is checked to make sure that no dataset name or listfile
    names are given yet. If all is well (i.e. we still have a
    clean slate) the listfile name is stored for later use,
    otherwise a Usage exception is raised.         """         if not self.input_method is None:
        if self.input_method == "listfile":
            raise Usage("Please only feed me one list file")
        elif self.input_method == "dataset":
            raise Usage("Cannot specify both dataset and input list file")
        else:
            assert False, "Unknown input method `%s'" % self.input_method
    self.input_method = "listfile"
    self.input_name = value
    self.logger.info("Input method used: %s" % self.input_method)         # End of option_handler_listfile_name. @verbatim Specify where on CASTOR the output should go.

At the moment only output to CERN CASTOR is supported. Eventually the harvested results should go into the central place for DQM on CASTOR anyway.



Definition at line 1058 of file cmsHarvester.py.


 1058 


 1059     def option_handler_castor_dir(self, option, opt_str, value, parser):


 1060         """Specify where on CASTOR the output should go.


 1061 


 1062         At the moment only output to CERN CASTOR is


 1063         supported. Eventually the harvested results should go into the


 1064         central place for DQM on CASTOR anyway.


 1065 


 1066         """


 1067 


 1068         # Check format of specified CASTOR area.


 1069         castor_dir = value


 1070         #castor_dir = castor_dir.lstrip(os.path.sep)


 1071         castor_prefix = self.castor_prefix


 1072 


 1073         # Add a leading slash if necessary and clean up the path.


 1074         castor_dir = os.path.join(os.path.sep, castor_dir)


 1075         self.castor_base_dir = os.path.normpath(castor_dir)


 1076 


 1077         self.logger.info("CASTOR (base) area to be used: `%s'" % \


 1078                          self.castor_base_dir)


 1079 


 1080         # End of option_handler_castor_dir.




cmsHarvester.option_handler_castor_dir
def option_handler_castor_dir
def option_handler_dataset_name(self, option, opt_str, value, parser): &quot;&quot;"Specify the name(s) of the ...
Definition: cmsHarvester.py:1058











      

        

          def cmsHarvester.option_handler_crab_submission 
          (
           
          self, 
        

        

          
          
           
          option, 
        

        

          
          
           
          opt_str, 
        

        

          
          
           
          value, 
        

        

          
          
           
          parser 
        

        

          
          )
          
        

      




Crab jobs are not created and
    "submitted automatically",

 

Definition at line 1128 of file cmsHarvester.py.


 1128 


 1129     def option_handler_crab_submission(self, option, opt_str, value, parser):


 1130         """Crab jobs are not created and


 1131     "submitted automatically",


 1132         """


 1133         self.crab_submission = True


 1134 


 1135         # End of option_handler_crab_submission.

       


cmsHarvester.option_handler_crab_submission
def option_handler_crab_submission
Definition: cmsHarvester.py:1128











      

        

          def cmsHarvester.option_handler_list_types 
          (
           
          self, 
        

        

          
          
           
          option, 
        

        

          
          
           
          opt_str, 
        

        

          
          
           
          value, 
        

        

          
          
           
          parser 
        

        

          
          )
          
        

      




List all harvesting types and their mappings.

This lists all implemented harvesting types with their
corresponding mappings to sequence names. This had to be
separated out from the help since it depends on the CMSSW
version and was making things a bit of a mess.

NOTE: There is no way (at least not that I could come up with)
to code this in a neat generic way that can be read both by
this method and by setup_harvesting_info(). Please try hard to
keep these two methods in sync!
 

Definition at line 1150 of file cmsHarvester.py.


 1150 


 1151     def option_handler_list_types(self, option, opt_str, value, parser):


 1152         """List all harvesting types and their mappings.


 1153 


 1154         This lists all implemented harvesting types with their


 1155         corresponding mappings to sequence names. This had to be


 1156         separated out from the help since it depends on the CMSSW


 1157         version and was making things a bit of a mess.


 1158 


 1159         NOTE: There is no way (at least not that I could come up with)


 1160         to code this in a neat generic way that can be read both by


 1161         this method and by setup_harvesting_info(). Please try hard to


 1162         keep these two methods in sync!


 1163 


 1164         """


 1165 


 1166         sep_line = "-" * 50


 1167         sep_line_short = "-" * 20


 1168 


 1169         print sep_line


 1170         print "The following harvesting types are available:"


 1171         print sep_line


 1172 


 1173         print "`RelVal' maps to:"


 1174         print "  pre-3_3_0           : HARVESTING:validationHarvesting"


 1175         print "  3_4_0_pre2 and later: HARVESTING:validationHarvesting+dqmHarvesting"


 1176         print "  Exceptions:"


 1177         print "    3_3_0_pre1-4        : HARVESTING:validationHarvesting"


 1178         print "    3_3_0_pre6          : HARVESTING:validationHarvesting"


 1179         print "    3_4_0_pre1          : HARVESTING:validationHarvesting"


 1180 


 1181         print sep_line_short


 1182 


 1183         print "`RelValFS' maps to:"


 1184         print "  always              : HARVESTING:validationHarvestingFS"


 1185 


 1186         print sep_line_short


 1187 


 1188         print "`MC' maps to:"


 1189         print "    always          : HARVESTING:validationprodHarvesting"


 1190 


 1191         print sep_line_short


 1192 


 1193         print "`DQMOffline' maps to:"


 1194         print "  always              : HARVESTING:dqmHarvesting"


 1195 


 1196         print sep_line


 1197 


 1198         # We're done, let's quit. (This is the same thing optparse


 1199         # does after printing the help.)


 1200         raise SystemExit


 1201 


 1202         # End of option_handler_list_types.




cmsHarvester.option_handler_list_types
def option_handler_list_types
Definition: cmsHarvester.py:1150











      

        

          def cmsHarvester.option_handler_no_t1access 
          (
           
          self, 
        

        

          
          
           
          option, 
        

        

          
          
           
          opt_str, 
        

        

          
          
           
          value, 
        

        

          
          
           
          parser 
        

        

          
          )
          
        

      




Set the self.no_t1access flag to try and create jobs that
run without special `t1access' role.
 

Definition at line 1083 of file cmsHarvester.py.


 1083 


 1084     def option_handler_no_t1access(self, option, opt_str, value, parser):


 1085         """Set the self.no_t1access flag to try and create jobs that


 1086         run without special `t1access' role.


 1087 


 1088         """


 1089 


 1090         self.non_t1access = True


 1091 


 1092         self.logger.warning("Running in `non-t1access' mode. " \


 1093                             "Will try to create jobs that run " \


 1094                             "without special rights but no " \


 1095                             "further promises...")


 1096 


 1097         # End of option_handler_no_t1access.

       


cmsHarvester.option_handler_no_t1access
def option_handler_no_t1access
Definition: cmsHarvester.py:1083











      

        

          def cmsHarvester.option_handler_preferred_site 
          (
           
          self, 
        

        

          
          
           
          option, 
        

        

          
          
           
          opt_str, 
        

        

          
          
           
          value, 
        

        

          
          
           
          parser 
        

        

          
          )
          
        

      





Definition at line 1144 of file cmsHarvester.py.


 1144 


 1145     def option_handler_preferred_site(self, option, opt_str, value, parser):


 1146 


 1147         self.preferred_site = value




cmsHarvester.option_handler_preferred_site
def option_handler_preferred_site
Definition: cmsHarvester.py:1144











      

        

          def cmsHarvester.option_handler_saveByLumiSection 
          (
           
          self, 
        

        

          
          
           
          option, 
        

        

          
          
           
          opt_str, 
        

        

          
          
           
          value, 
        

        

          
          
           
          parser 
        

        

          
          )
          
        

      




Set process.dqmSaver.saveByLumiSectiont=1 in cfg harvesting file

 

Definition at line 1116 of file cmsHarvester.py.


 1116 


 1117     def option_handler_saveByLumiSection(self, option, opt_str, value, parser):


 1118         """Set process.dqmSaver.saveByLumiSectiont=1 in cfg harvesting file


 1119         """


 1120         self.saveByLumiSection = True


 1121 


 1122         self.logger.warning("waning concerning saveByLumiSection option")


 1123 


 1124         # End of option_handler_saveByLumiSection.


 1125 




cmsHarvester.option_handler_saveByLumiSection
def option_handler_saveByLumiSection
Definition: cmsHarvester.py:1116











      

        

          def cmsHarvester.option_handler_sites 
          (
           
          self, 
        

        

          
          
           
          option, 
        

        

          
          
           
          opt_str, 
        

        

          
          
           
          value, 
        

        

          
          
           
          parser 
        

        

          
          )
          
        

      





Definition at line 1138 of file cmsHarvester.py.


 1138 


 1139     def option_handler_sites(self, option, opt_str, value, parser):


 1140 


 1141         self.nr_max_sites = value




cmsHarvester.option_handler_sites
def option_handler_sites
Definition: cmsHarvester.py:1138











      

        

          def cmsHarvester.parse_cmd_line_options 
          (
           
          self)
          
        

      





Definition at line 1869 of file cmsHarvester.py.


 1869 


 1870     def parse_cmd_line_options(self):


 1871 


 1872         # Set up the command line parser. Note that we fix up the help


 1873         # formatter so that we can add some text pointing people to


 1874         # the Twiki etc.


 1875         parser = optparse.OptionParser(version="%s %s" % \


 1876                                        ("%prog", self.version),


 1877                                        formatter=CMSHarvesterHelpFormatter())


 1878 


 1879         self.option_parser = parser


 1880 


 1881         # The debug switch.


 1882         parser.add_option("-d", "--debug",


 1883                           help="Switch on debug mode",


 1884                           action="callback",


 1885                           callback=self.option_handler_debug)


 1886 


 1887         # The quiet switch.


 1888         parser.add_option("-q", "--quiet",


 1889                           help="Be less verbose",


 1890                           action="callback",


 1891                           callback=self.option_handler_quiet)


 1892 


 1893         # The force switch. If this switch is used sanity checks are


 1894         # performed but failures do not lead to aborts. Use with care.


 1895         parser.add_option("", "--force",


 1896                           help="Force mode. Do not abort on sanity check "


 1897                           "failures",


 1898                           action="callback",


 1899                           callback=self.option_handler_force)


 1900 


 1901         # Choose between the different kinds of harvesting.


 1902         parser.add_option("", "--harvesting_type",


 1903                           help="Harvesting type: %s" % \


 1904                           ", ".join(self.harvesting_types),


 1905                           action="callback",


 1906                           callback=self.option_handler_harvesting_type,


 1907                           type="string",


 1908                           metavar="HARVESTING_TYPE")


 1909 


 1910         # Choose between single-step and two-step mode.


 1911         parser.add_option("", "--harvesting_mode",


 1912                           help="Harvesting mode: %s (default = %s)" % \


 1913                           (", ".join(self.harvesting_modes),


 1914                           self.harvesting_mode_default),


 1915                           action="callback",


 1916                           callback=self.option_handler_harvesting_mode,


 1917                           type="string",


 1918                           metavar="HARVESTING_MODE")


 1919 


 1920         # Override the GlobalTag chosen by the cmsHarvester.


 1921         parser.add_option("", "--globaltag",


 1922                           help="GlobalTag to use. Default is the ones " \


 1923                           "the dataset was created with for MC, for data" \


 1924                           "a GlobalTag has to be specified.",


 1925                           action="callback",


 1926                           callback=self.option_handler_globaltag,


 1927                           type="string",


 1928                           metavar="GLOBALTAG")


 1929 


 1930         # Allow switching off of reference histograms.


 1931         parser.add_option("", "--no-ref-hists",


 1932                           help="Don't use any reference histograms",


 1933                           action="callback",


 1934                           callback=self.option_handler_no_ref_hists)


 1935 


 1936         # Allow the default (i.e. the one that should be used)


 1937         # Frontier connection to be overridden.


 1938         parser.add_option("", "--frontier-connection",


 1939                           help="Use this Frontier connection to find " \


 1940                           "GlobalTags and LocalTags (for reference " \


 1941                           "histograms).\nPlease only use this for " \


 1942                           "testing.",


 1943                           action="callback",


 1944                           callback=self.option_handler_frontier_connection,


 1945                           type="string",


 1946                           metavar="FRONTIER")


 1947 


 1948         # Similar to the above but specific to the Frontier connection


 1949         # to be used for the GlobalTag.


 1950         parser.add_option("", "--frontier-connection-for-globaltag",


 1951                           help="Use this Frontier connection to find " \


 1952                           "GlobalTags.\nPlease only use this for " \


 1953                           "testing.",


 1954                           action="callback",


 1955                           callback=self.option_handler_frontier_connection,


 1956                           type="string",


 1957                           metavar="FRONTIER")


 1958 


 1959         # Similar to the above but specific to the Frontier connection


 1960         # to be used for the reference histograms.


 1961         parser.add_option("", "--frontier-connection-for-refhists",


 1962                           help="Use this Frontier connection to find " \


 1963                           "LocalTags (for reference " \


 1964                           "histograms).\nPlease only use this for " \


 1965                           "testing.",


 1966                           action="callback",


 1967                           callback=self.option_handler_frontier_connection,


 1968                           type="string",


 1969                           metavar="FRONTIER")


 1970 


 1971         # Option to specify the name (or a regexp) of the dataset(s)


 1972         # to be used.


 1973         parser.add_option("", "--dataset",


 1974                           help="Name (or regexp) of dataset(s) to process",


 1975                           action="callback",


 1976                           #callback=self.option_handler_dataset_name,


 1977                           callback=self.option_handler_input_spec,


 1978                           type="string",


 1979                           #dest="self.input_name",


 1980                           metavar="DATASET")


 1981 


 1982         # Option to specify the name (or a regexp) of the dataset(s)


 1983         # to be ignored.


 1984         parser.add_option("", "--dataset-ignore",


 1985                           help="Name (or regexp) of dataset(s) to ignore",


 1986                           action="callback",


 1987                           callback=self.option_handler_input_spec,


 1988                           type="string",


 1989                           metavar="DATASET-IGNORE")


 1990 


 1991         # Option to specify the name (or a regexp) of the run(s)


 1992         # to be used.


 1993         parser.add_option("", "--runs",


 1994                           help="Run number(s) to process",


 1995                           action="callback",


 1996                           callback=self.option_handler_input_spec,


 1997                           type="string",


 1998                           metavar="RUNS")                   


 1999 


 2000         # Option to specify the name (or a regexp) of the run(s)


 2001         # to be ignored.


 2002         parser.add_option("", "--runs-ignore",


 2003                           help="Run number(s) to ignore",


 2004                           action="callback",


 2005                           callback=self.option_handler_input_spec,


 2006                           type="string",


 2007                           metavar="RUNS-IGNORE")


 2008 


 2009         # Option to specify a file containing a list of dataset names


 2010         # (or regexps) to be used.


 2011         parser.add_option("", "--datasetfile",


 2012                           help="File containing list of dataset names " \


 2013                           "(or regexps) to process",


 2014                           action="callback",


 2015                           #callback=self.option_handler_listfile_name,


 2016                           callback=self.option_handler_input_spec,


 2017                           type="string",


 2018                           #dest="self.input_name",


 2019                           metavar="DATASETFILE")


 2020 


 2021         # Option to specify a file containing a list of dataset names


 2022         # (or regexps) to be ignored.


 2023         parser.add_option("", "--datasetfile-ignore",


 2024                           help="File containing list of dataset names " \


 2025                           "(or regexps) to ignore",


 2026                           action="callback",


 2027                           callback=self.option_handler_input_spec,


 2028                           type="string",


 2029                           metavar="DATASETFILE-IGNORE")


 2030 


 2031         # Option to specify a file containing a list of runs to be


 2032         # used.


 2033         parser.add_option("", "--runslistfile",


 2034                           help="File containing list of run numbers " \


 2035                           "to process",


 2036                           action="callback",


 2037                           callback=self.option_handler_input_spec,


 2038                           type="string",


 2039                           metavar="RUNSLISTFILE")


 2040 


 2041         # Option to specify a file containing a list of runs


 2042         # to be ignored.


 2043         parser.add_option("", "--runslistfile-ignore",


 2044                           help="File containing list of run numbers " \


 2045                           "to ignore",


 2046                           action="callback",


 2047                           callback=self.option_handler_input_spec,


 2048                           type="string",


 2049                           metavar="RUNSLISTFILE-IGNORE")


 2050 

        # Option to specify a Jsonfile contaning a list of runs


cmsHarvester.parse_cmd_line_options
def parse_cmd_line_options
Definition: cmsHarvester.py:1869


join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18


cmsHarvester.CMSHarvesterHelpFormatter
Helper class: CMSHarvesterHelpFormatter. 
Definition: cmsHarvester.py:149











      

        

          def cmsHarvester.pick_a_site 
          (
           
          self, 
        

        

          
          
           
          sites, 
        

        

          
          
           
          cmssw_version 
        

        

          
          )
          
        

      





Definition at line 1705 of file cmsHarvester.py.


 1705 


 1706     def pick_a_site(self, sites, cmssw_version):














      

        

          def cmsHarvester.process_dataset_ignore_list 
          (
           
          self)
          
        

      




Update the list of datasets taking into account the ones to
ignore.

Both lists have been generated before from DBS and both are
assumed to be unique.

NOTE: The advantage of creating the ignore list from DBS (in
case a regexp is given) and matching that instead of directly
matching the ignore criterion against the list of datasets (to
consider) built from DBS is that in the former case we're sure
that all regexps are treated exactly as DBS would have done
without the cmsHarvester.

NOTE: This only removes complete samples. Exclusion of single
runs is done by the book keeping. So the assumption is that a
user never wants to harvest just part (i.e. n out of N runs)
of a sample.
 

Definition at line 3566 of file cmsHarvester.py.


 3566 


 3567     def process_dataset_ignore_list(self):


 3568         """Update the list of datasets taking into account the ones to


 3569         ignore.


 3570 


 3571         Both lists have been generated before from DBS and both are


 3572         assumed to be unique.


 3573 


 3574         NOTE: The advantage of creating the ignore list from DBS (in


 3575         case a regexp is given) and matching that instead of directly


 3576         matching the ignore criterion against the list of datasets (to


 3577         consider) built from DBS is that in the former case we're sure


 3578         that all regexps are treated exactly as DBS would have done


 3579         without the cmsHarvester.


 3580 


 3581         NOTE: This only removes complete samples. Exclusion of single


 3582         runs is done by the book keeping. So the assumption is that a


 3583         user never wants to harvest just part (i.e. n out of N runs)


 3584         of a sample.


 3585 


 3586         """


 3587 


 3588         self.logger.info("Processing list of datasets to ignore...")


 3589 


 3590         self.logger.debug("Before processing ignore list there are %d " \


 3591                           "datasets in the list to be processed" % \


 3592                           len(self.datasets_to_use))


 3593 


 3594         # Simple approach: just loop and search.


 3595         dataset_names_filtered = copy.deepcopy(self.datasets_to_use)


 3596         for dataset_name in self.datasets_to_use.keys():


 3597             if dataset_name in self.datasets_to_ignore.keys():


 3598                 del dataset_names_filtered[dataset_name]


 3599 


 3600         self.logger.info("  --> Removed %d dataset(s)" % \


 3601                          (len(self.datasets_to_use) -


 3602                           len(dataset_names_filtered)))


 3603 


 3604         self.datasets_to_use = dataset_names_filtered


 3605 


 3606         self.logger.debug("After processing ignore list there are %d " \


 3607                           "datasets in the list to be processed" % \


 3608                           len(self.datasets_to_use))




cmsHarvester.process_dataset_ignore_list
def process_dataset_ignore_list
Definition: cmsHarvester.py:3566











      

        

          def cmsHarvester.process_runs_use_and_ignore_lists 
          (
           
          self)
          
        

      





Definition at line 3613 of file cmsHarvester.py.


 3613 


 3614     def process_runs_use_and_ignore_lists(self):


 3615 


 3616         self.logger.info("Processing list of runs to use and ignore...")


 3617 


 3618         # This basically adds all runs in a dataset to be processed,


 3619         # except for any runs that are not specified in the `to use'


 3620         # list and any runs that are specified in the `to ignore'


 3621         # list.


 3622 


 3623         # NOTE: It is assumed that those lists make sense. The input


 3624         # should be checked against e.g. overlapping `use' and


 3625         # `ignore' lists.


 3626 


 3627         runs_to_use = self.runs_to_use


 3628         runs_to_ignore = self.runs_to_ignore


 3629 


 3630         for dataset_name in self.datasets_to_use:


 3631             runs_in_dataset = self.datasets_information[dataset_name]["runs"]


 3632 


 3633             # First some sanity checks.


 3634             runs_to_use_tmp = []


 3635             for run in runs_to_use:


 3636                 if not run in runs_in_dataset:


 3637                     self.logger.warning("Dataset `%s' does not contain " \


 3638                                         "requested run %d " \


 3639                                         "--> ignoring `use' of this run" % \


 3640                                         (dataset_name, run))


 3641                 else:


 3642                     runs_to_use_tmp.append(run)


 3643 


 3644             if len(runs_to_use) > 0:


 3645                 runs = runs_to_use_tmp


 3646                 self.logger.info("Using %d out of %d runs " \


 3647                                  "of dataset `%s'" % \


 3648                                  (len(runs), len(runs_in_dataset),


 3649                                   dataset_name))


 3650             else:


 3651                 runs = runs_in_dataset


 3652 


 3653             if len(runs_to_ignore) > 0:


 3654                 runs_tmp = []


 3655                 for run in runs:


 3656                     if not run in runs_to_ignore:


 3657                         runs_tmp.append(run)


 3658                 self.logger.info("Ignoring %d out of %d runs " \


 3659                                  "of dataset `%s'" % \


 3660                                  (len(runs)- len(runs_tmp),


 3661                                   len(runs_in_dataset),


 3662                                   dataset_name))


 3663                 runs = runs_tmp


 3664 


 3665             if self.todofile != "YourToDofile.txt":


 3666         runs_todo = []


 3667                 print "Reading runs from file /afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/harvesting/%s" %self.todofile


 3668                 cmd="grep %s /afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/harvesting/%s | cut -f5 -d' '" %(dataset_name,self.todofile)


 3669                 (status, output)=commands.getstatusoutput(cmd)


 3670         for run in runs:


 3671             run_str="%s" %run


 3672             if run_str in output:


 3673             runs_todo.append(run)


 3674                 self.logger.info("Using %d runs " \


 3675                                  "of dataset `%s'" % \


 3676                                  (len(runs_todo),


 3677                                   dataset_name))


 3678         runs=runs_todo


 3679 


 3680         Json_runs = []


 3681         if self.Jsonfilename != "YourJSON.txt":


 3682         good_runs = []


 3683         self.Jsonlumi = True


 3684         # We were passed a Jsonfile containing a dictionary of


 3685         # run/lunisection-pairs


 3686         self.logger.info("Reading runs and lumisections from file `%s'" % \


 3687                 self.Jsonfilename)


 3688         try:


 3689             Jsonfile = open(self.Jsonfilename, "r")


 3690             for names in Jsonfile:


 3691             dictNames= eval(str(names))


 3692             for key in dictNames:


 3693                 intkey=int(key)


 3694                 Json_runs.append(intkey)


 3695             Jsonfile.close()


 3696         except IOError:


 3697             msg = "ERROR: Could not open Jsonfile `%s'" % \


 3698               input_name


 3699             self.logger.fatal(msg)


 3700             raise Error(msg)


 3701         for run in runs:


 3702             if run in Json_runs:


 3703             good_runs.append(run)


 3704         self.logger.info("Using %d runs " \


 3705                                  "of dataset `%s'" % \


 3706                                  (len(good_runs),


 3707                                   dataset_name))


 3708         runs=good_runs


 3709             if (self.Jsonrunfilename != "YourJSON.txt") and (self.Jsonfilename == "YourJSON.txt"):


 3710         good_runs = []


 3711                 # We were passed a Jsonfile containing a dictionary of


 3712         # run/lunisection-pairs


 3713             self.logger.info("Reading runs from file `%s'" % \


 3714                 self.Jsonrunfilename)


 3715         try:


 3716             Jsonfile = open(self.Jsonrunfilename, "r")


 3717             for names in Jsonfile:


 3718                     dictNames= eval(str(names))


 3719             for key in dictNames:


 3720                 intkey=int(key)


 3721                 Json_runs.append(intkey)


 3722             Jsonfile.close()


 3723         except IOError:


 3724             msg = "ERROR: Could not open Jsonfile `%s'" % \


 3725               input_name


 3726             self.logger.fatal(msg)


 3727             raise Error(msg)


 3728             for run in runs:


 3729                 if run in Json_runs: 


 3730                         good_runs.append(run)


 3731                 self.logger.info("Using %d runs " \


 3732                                  "of dataset `%s'" % \


 3733                                  (len(good_runs),


 3734                                   dataset_name))


 3735         runs=good_runs


 3736 


 3737             self.datasets_to_use[dataset_name] = runs


 3738 


 3739         # End of process_runs_use_and_ignore_lists().




cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.process_runs_use_and_ignore_lists
def process_runs_use_and_ignore_lists
Definition: cmsHarvester.py:3613











      

        

          def cmsHarvester.ref_hist_mappings_needed 
          (
           
          self, 
        

        

          
          
           
          dataset_name = None 
        

        

          
          )
          
        

      




Check if we need to load and check the reference mappings.

For data the reference histograms should be taken
automatically from the GlobalTag, so we don't need any
mappings. For RelVals we need to know a mapping to be used in
the es_prefer code snippet (different references for each of
the datasets.)

WARNING: This implementation is a bit convoluted.
 

Definition at line 5172 of file cmsHarvester.py.


 5172 


 5173     def ref_hist_mappings_needed(self, dataset_name=None):


 5174         """Check if we need to load and check the reference mappings.


 5175 


 5176         For data the reference histograms should be taken


 5177         automatically from the GlobalTag, so we don't need any


 5178         mappings. For RelVals we need to know a mapping to be used in


 5179         the es_prefer code snippet (different references for each of


 5180         the datasets.)


 5181 


 5182         WARNING: This implementation is a bit convoluted.


 5183 


 5184         """


 5185 


 5186         # If no dataset name given, do everything, otherwise check


 5187         # only this one dataset.


 5188         if not dataset_name is None:


 5189             data_type = self.datasets_information[dataset_name] \


 5190                         ["datatype"]


 5191             mappings_needed = (data_type == "mc")


 5192             # DEBUG DEBUG DEBUG


 5193             if not mappings_needed:


 5194                 assert data_type == "data"


 5195             # DEBUG DEBUG DEBUG end


 5196         else:


 5197             tmp = [self.ref_hist_mappings_needed(dataset_name) \


 5198                    for dataset_name in \


 5199                    self.datasets_information.keys()]


 5200             mappings_needed = (True in tmp)


 5201 


 5202         # End of ref_hist_mappings_needed.


 5203         return mappings_needed




cmsHarvester.ref_hist_mappings_needed
def ref_hist_mappings_needed
Definition: cmsHarvester.py:5172











      

        

          def cmsHarvester.run 
          (
           
          self)
          
        

      





Definition at line 5524 of file cmsHarvester.py.



References update.


 5524 


 5525     def run(self):


 5526         "Main entry point of the CMS harvester."


 5527 


 5528         # Start with a positive thought.


 5529         exit_code = 0


 5530 


 5531         try:


 5532 


 5533             try:


 5534 


 5535                 # Parse all command line options and arguments


 5536                 self.parse_cmd_line_options()


 5537                 # and check that they make sense.


 5538                 self.check_input_status()


 5539 


 5540                 # Check if CMSSW is setup.


 5541                 self.check_cmssw()


 5542 


 5543                 # Check if DBS is setup,


 5544                 self.check_dbs()


 5545                 # and if all is fine setup the Python side.


 5546                 self.setup_dbs()


 5547 


 5548                 # Fill our dictionary with all the required info we


 5549                 # need to understand harvesting jobs. This needs to be


 5550                 # done after the CMSSW version is known.


 5551                 self.setup_harvesting_info()


 5552 


 5553                 # Obtain list of dataset names to consider


 5554                 self.build_dataset_use_list()


 5555                 # and the list of dataset names to ignore.


 5556                 self.build_dataset_ignore_list()


 5557 


 5558                 # The same for the runs lists (if specified).


 5559                 self.build_runs_use_list()


 5560                 self.build_runs_ignore_list()


 5561 


 5562                 # Process the list of datasets to ignore and fold that


 5563                 # into the list of datasets to consider.


 5564                 # NOTE: The run-based selection is done later since


 5565                 # right now we don't know yet which runs a dataset


 5566                 # contains.


 5567                 self.process_dataset_ignore_list()


 5568 


 5569                 # Obtain all required information on the datasets,


 5570                 # like run numbers and GlobalTags.


 5571                 self.build_datasets_information()


 5572 


 5573                 if self.use_ref_hists and \


 5574                        self.ref_hist_mappings_needed():


 5575                     # Load the dataset name to reference histogram


 5576                     # name mappings from file.


 5577                     self.load_ref_hist_mappings()


 5578                     # Now make sure that for all datasets we want to


 5579                     # process there is a reference defined. Otherwise


 5580                     # just bomb out before wasting any more time.


 5581                     self.check_ref_hist_mappings()


 5582                 else:


 5583                     self.logger.info("No need to load reference " \


 5584                                      "histogram mappings file")


 5585 

                # OBSOLETE OBSOLETE OBSOLETE












      

        

          def cmsHarvester.setup_dbs 
          (
           
          self)
          
        

      





Now we try to do a very simple DBS search.




If that works 


instead of giving us the `Unsupported API call' crap, we


should be good to go.


NOTE: Not ideal, I know, but it reduces the amount of


complaints I get...


cmd = "dbs search --query=\"find dataset where dataset = impossible"" (status, output) = commands.getstatusoutput(cmd) pdb.set_trace() if status != 0 or \ output.lower().find("unsupported api call") > -1: self.logger.fatal("It seems DBS is not setup...") self.logger.fatal("  %s returns crap:" % cmd) for line in output.split("\n"): self.logger.fatal("  %s" % line) raise Error("ERROR: DBS needs to be setup first!") 


Setup the Python side of DBS.

For more information see the DBS Python API documentation:
https://twiki.cern.ch/twiki/bin/view/CMS/DBSApiDocumentation
 

Definition at line 2392 of file cmsHarvester.py.


 2392 


 2393     def setup_dbs(self):


 2394         """Setup the Python side of DBS.


 2395 


 2396         For more information see the DBS Python API documentation:


 2397         https://twiki.cern.ch/twiki/bin/view/CMS/DBSApiDocumentation


 2398 


 2399         """


 2400 


 2401         try:


 2402             args={}


 2403             args["url"]= "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/" \


 2404                          "servlet/DBSServlet"


 2405             api = DbsApi(args)


 2406             self.dbs_api = api


 2407 


 2408         except DBSAPI.dbsApiException.DbsApiException, ex:


 2409             self.logger.fatal("Caught DBS API exception %s: %s "  % \


 2410                               (ex.getClassName(), ex.getErrorMessage()))


 2411             if ex.getErrorCode() not in (None, ""):


 2412                 logger.debug("DBS exception error code: ", ex.getErrorCode())


 2413             raise


 2414 


 2415         # End of setup_dbs.




cmsHarvester.setup_dbs
def setup_dbs
Now we try to do a very simple DBS search.
Definition: cmsHarvester.py:2392











      

        

          def cmsHarvester.setup_harvesting_info 
          (
           
          self)
          
        

      




Fill our dictionary with all info needed to understand
harvesting.

This depends on the CMSSW version since at some point the
names and sequences were modified.

NOTE: There is no way (at least not that I could come up with)
to code this in a neat generic way that can be read both by
this method and by option_handler_list_types(). Please try
hard to keep these two methods in sync!
 

Definition at line 1205 of file cmsHarvester.py.


 1205 


 1206     def setup_harvesting_info(self):


 1207         """Fill our dictionary with all info needed to understand


 1208         harvesting.


 1209 


 1210         This depends on the CMSSW version since at some point the


 1211         names and sequences were modified.


 1212 


 1213         NOTE: There is no way (at least not that I could come up with)


 1214         to code this in a neat generic way that can be read both by


 1215         this method and by option_handler_list_types(). Please try


 1216         hard to keep these two methods in sync!


 1217 


 1218         """


 1219 


 1220         assert not self.cmssw_version is None, \


 1221                "ERROR setup_harvesting() requires " \


 1222                "self.cmssw_version to be set!!!"


 1223 


 1224         harvesting_info = {}


 1225 


 1226         # This is the version-independent part.


 1227         harvesting_info["DQMOffline"] = {}


 1228         harvesting_info["DQMOffline"]["beamspot"] = None


 1229         harvesting_info["DQMOffline"]["eventcontent"] = None


 1230         harvesting_info["DQMOffline"]["harvesting"] = "AtRunEnd"


 1231 


 1232         harvesting_info["RelVal"] = {}


 1233         harvesting_info["RelVal"]["beamspot"] = None


 1234         harvesting_info["RelVal"]["eventcontent"] = None


 1235         harvesting_info["RelVal"]["harvesting"] = "AtRunEnd"


 1236 


 1237         harvesting_info["RelValFS"] = {}


 1238         harvesting_info["RelValFS"]["beamspot"] = None


 1239         harvesting_info["RelValFS"]["eventcontent"] = None


 1240         harvesting_info["RelValFS"]["harvesting"] = "AtRunEnd"


 1241 


 1242         harvesting_info["MC"] = {}


 1243         harvesting_info["MC"]["beamspot"] = None


 1244         harvesting_info["MC"]["eventcontent"] = None


 1245         harvesting_info["MC"]["harvesting"] = "AtRunEnd"


 1246 


 1247         # This is the version-dependent part. And I know, strictly


 1248         # speaking it's not necessary to fill in all three types since


 1249         # in a single run we'll only use one type anyway. This does


 1250         # look more readable, however, and required less thought from


 1251         # my side when I put this together.


 1252 


 1253         # DEBUG DEBUG DEBUG


 1254         # Check that we understand our own version naming.


 1255         assert self.cmssw_version.startswith("CMSSW_")


 1256         # DEBUG DEBUG DEBUG end


 1257 


 1258         version = self.cmssw_version[6:]


 1259 


 1260         #----------


 1261 


 1262         # RelVal


 1263         step_string = None


 1264         if version < "3_3_0":


 1265             step_string = "validationHarvesting"


 1266         elif version in ["3_3_0_pre1", "3_3_0_pre2",


 1267                          "3_3_0_pre3", "3_3_0_pre4",


 1268                          "3_3_0_pre6", "3_4_0_pre1"]:


 1269             step_string = "validationHarvesting"


 1270         else:


 1271             step_string = "validationHarvesting+dqmHarvesting"


 1272 


 1273         harvesting_info["RelVal"]["step_string"] = step_string


 1274 


 1275         # DEBUG DEBUG DEBUG


 1276         # Let's make sure we found something.


 1277         assert not step_string is None, \


 1278                "ERROR Could not decide a RelVal harvesting sequence " \


 1279                "for CMSSW version %s" % self.cmssw_version


 1280         # DEBUG DEBUG DEBUG end


 1281 


 1282         #----------


 1283 


 1284         # RelVal


 1285         step_string = "validationHarvestingFS"


 1286 


 1287         harvesting_info["RelValFS"]["step_string"] = step_string


 1288 


 1289         #----------


 1290 


 1291         # MC


 1292         step_string = "validationprodHarvesting"


 1293 


 1294         harvesting_info["MC"]["step_string"] = step_string


 1295 


 1296         # DEBUG DEBUG DEBUG


 1297         # Let's make sure we found something.


 1298         assert not step_string is None, \


 1299                "ERROR Could not decide a MC harvesting " \


 1300                "sequence for CMSSW version %s" % self.cmssw_version


 1301         # DEBUG DEBUG DEBUG end


 1302 


 1303         #----------


 1304 


 1305         # DQMOffline


 1306         step_string = "dqmHarvesting"


 1307 


 1308         harvesting_info["DQMOffline"]["step_string"] = step_string


 1309 


 1310         #----------


 1311 


 1312         self.harvesting_info = harvesting_info


 1313 


 1314         self.logger.info("Based on the CMSSW version (%s) " \


 1315                          "I decided to use the `HARVESTING:%s' " \


 1316                          "sequence for %s harvesting" % \


 1317                          (self.cmssw_version,


 1318                           self.harvesting_info[self.harvesting_type]["step_string"],


 1319                           self.harvesting_type))


 1320 


 1321         # End of setup_harvesting_info.




cmsHarvester.setup_harvesting_info
def setup_harvesting_info
Definition: cmsHarvester.py:1205











      

        

          def cmsHarvester.show_exit_message 
          (
           
          self)
          
        

      




Tell the user what to do now, after this part is done.

This should provide the user with some (preferably
copy-pasteable) instructions on what to do now with the setups
and files that have been created.
 

Definition at line 5471 of file cmsHarvester.py.


 5471 


 5472     def show_exit_message(self):


 5473         """Tell the user what to do now, after this part is done.


 5474 


 5475         This should provide the user with some (preferably


 5476         copy-pasteable) instructions on what to do now with the setups


 5477         and files that have been created.


 5478 


 5479         """


 5480 


 5481         # TODO TODO TODO


 5482         # This could be improved a bit.


 5483         # TODO TODO TODO end


 5484 


 5485         sep_line = "-" * 60


 5486 


 5487         self.logger.info("")


 5488         self.logger.info(sep_line)


 5489         self.logger.info("  Configuration files have been created.")


 5490         self.logger.info("  From here on please follow the usual CRAB instructions.")


 5491         self.logger.info("  Quick copy-paste instructions are shown below.")


 5492         self.logger.info(sep_line)


 5493 


 5494         self.logger.info("")


 5495         self.logger.info("    Create all CRAB jobs:")


 5496         self.logger.info("      multicrab -create")


 5497         self.logger.info("")


 5498         self.logger.info("    Submit all CRAB jobs:")


 5499         self.logger.info("      multicrab -submit")


 5500         self.logger.info("")


 5501         self.logger.info("    Check CRAB status:")


 5502         self.logger.info("      multicrab -status")


 5503         self.logger.info("")


 5504 


 5505         self.logger.info("")


 5506         self.logger.info("  For more information please see the CMS Twiki:")


 5507         self.logger.info("    %s" % twiki_url)


 5508         self.logger.info(sep_line)


 5509 


 5510         # If there were any jobs for which we could not find a


 5511         # matching site show a warning message about that.


 5512         if not self.all_sites_found:


 5513             self.logger.warning("  For some of the jobs no matching " \


 5514                                 "site could be found")


 5515             self.logger.warning("  --> please scan your multicrab.cfg" \


 5516                                 "for occurrences of `%s'." % \


 5517                                 self.no_matching_site_found_str)


 5518             self.logger.warning("      You will have to fix those " \


 5519                                 "by hand, sorry.")


 5520 


 5521         # End of show_exit_message.














      

        

          def cmsHarvester.singlify_datasets 
          (
           
          self)
          
        

      




Remove all but the largest part of all datasets.

This allows us to harvest at least part of these datasets
using single-step harvesting until the two-step approach
works.
 

Definition at line 3742 of file cmsHarvester.py.



References bookConverter.max, and makeHLTPrescaleTable.values.


 3742 


 3743     def singlify_datasets(self):


 3744         """Remove all but the largest part of all datasets.


 3745 


 3746         This allows us to harvest at least part of these datasets


 3747         using single-step harvesting until the two-step approach


 3748         works.


 3749 


 3750         """


 3751 


 3752         # DEBUG DEBUG DEBUG


 3753         assert self.harvesting_mode == "single-step-allow-partial"


 3754         # DEBUG DEBUG DEBUG end


 3755 


 3756         for dataset_name in self.datasets_to_use:


 3757             for run_number in self.datasets_information[dataset_name]["runs"]:


 3758                 max_events = max(self.datasets_information[dataset_name]["sites"][run_number].values())


 3759                 sites_with_max_events = [i[0] for i in self.datasets_information[dataset_name]["sites"][run_number].items() if i[1] == max_events]


 3760                 self.logger.warning("Singlifying dataset `%s', " \


 3761                                     "run %d" % \


 3762                                     (dataset_name, run_number))


 3763                 cmssw_version = self.datasets_information[dataset_name] \


 3764                                 ["cmssw_version"]


 3765                 selected_site = self.pick_a_site(sites_with_max_events,


 3766                                                  cmssw_version)


 3767 


 3768                 # Let's tell the user that we're manhandling this dataset.


 3769                 nevents_old = self.datasets_information[dataset_name]["num_events"][run_number]


 3770                 self.logger.warning("  --> " \


 3771                                     "only harvesting partial statistics: " \


 3772                                     "%d out of %d events (5.1%f%%) " \


 3773                                     "at site `%s'" % \


 3774                                     (max_events,


 3775                                      nevents_old,


 3776                                      100. * max_events / nevents_old,


 3777                                      selected_site))


 3778                 self.logger.warning("!!! Please note that the number of " \


 3779                                     "events in the output path name will " \


 3780                                     "NOT reflect the actual statistics in " \


 3781                                     "the harvested results !!!")


 3782 


 3783                 # We found the site with the highest statistics and


 3784                 # the corresponding number of events. (CRAB gets upset


 3785                 # if we ask for more events than there are at a given


 3786                 # site.) Now update this information in our main


 3787                 # datasets_information variable.


 3788                 self.datasets_information[dataset_name]["sites"][run_number] = {selected_site: max_events}


 3789                 self.datasets_information[dataset_name]["num_events"][run_number] = max_events


 3790                 #self.datasets_information[dataset_name]["sites"][run_number] = [selected_site]


 3791 


 3792         # End of singlify_datasets.
















      

        

          def cmsHarvester.write_crab_config 
          (
           
          self)
          
        

      





def create_harvesting_config(self, dataset_name): """Create the Python harvesting configuration for a given job. 


     NOTE: The reason to have a single harvesting configuration per
    sample is to be able to specify the GlobalTag corresponding to
    each sample. Since it has been decided that (apart from the
    prompt reco) datasets cannot contain runs with different
    GlobalTags, we don't need a harvesting config per run.         NOTE: This is the place where we distinguish between
    single-step and two-step harvesting modes (at least for the
    Python job configuration).         """         ###         if self.harvesting_mode == "single-step":
        config_contents = self.create_harvesting_config_single_step(dataset_name)
    elif self.harvesting_mode == "two-step":
        config_contents = self.create_harvesting_config_two_step(dataset_name)
    else:

 
Impossible harvesting mode, we should never get here.


assert False, "ERROR: unknown harvesting mode `%s'" % \ self.harvesting_mode ### # End of create_harvesting_config. return config_contents 


Write a CRAB job configuration Python file.
 

Definition at line 5048 of file cmsHarvester.py.



References mergeVDriftHistosByStation.file.


 5048 


 5049     def write_crab_config(self):


 5050         """Write a CRAB job configuration Python file.


 5051 


 5052         """


 5053 


 5054         self.logger.info("Writing CRAB configuration...")


 5055 


 5056         file_name_base = "crab.cfg"


 5057 


 5058         # Create CRAB configuration.


 5059         crab_contents = self.create_crab_config()


 5060 


 5061         # Write configuration to file.


 5062         crab_file_name = file_name_base


 5063         try:


 5064             crab_file = file(crab_file_name, "w")


 5065             crab_file.write(crab_contents)


 5066             crab_file.close()


 5067         except IOError:


 5068             self.logger.fatal("Could not write " \


 5069                               "CRAB configuration to file `%s'" % \


 5070                               crab_file_name)


 5071             raise Error("ERROR: Could not write to file `%s'!" % \


 5072                         crab_file_name)


 5073 


 5074         # End of write_crab_config.





cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.write_crab_config
def write_crab_config
def create_harvesting_config(self, dataset_name): &quot;&quot;"Create the Python harvesting configuration for a...
Definition: cmsHarvester.py:5048











      

        

          def cmsHarvester.write_harvesting_config 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      




Write a harvesting job configuration Python file.

NOTE: This knows nothing about single-step or two-step
harvesting. That's all taken care of by
create_harvesting_config.
 

Definition at line 5106 of file cmsHarvester.py.



References create_harvesting_config_file_name(), and mergeVDriftHistosByStation.file.


 5106 


 5107     def write_harvesting_config(self, dataset_name):


 5108         """Write a harvesting job configuration Python file.


 5109 


 5110         NOTE: This knows nothing about single-step or two-step


 5111         harvesting. That's all taken care of by


 5112         create_harvesting_config.


 5113 


 5114         """


 5115 


 5116         self.logger.debug("Writing harvesting configuration for `%s'..." % \


 5117                           dataset_name)


 5118 


 5119         # Create Python configuration.


 5120         config_contents = self.create_harvesting_config(dataset_name)


 5121 


 5122         # Write configuration to file.


 5123         config_file_name = self. \


 5124                            create_harvesting_config_file_name(dataset_name)


 5125         try:


 5126             config_file = file(config_file_name, "w")


 5127             config_file.write(config_contents)


 5128             config_file.close()


 5129         except IOError:


 5130             self.logger.fatal("Could not write " \


 5131                               "harvesting configuration to file `%s'" % \


 5132                               config_file_name)


 5133             raise Error("ERROR: Could not write to file `%s'!" % \


 5134                         config_file_name)


 5135 


 5136         # End of write_harvesting_config.





cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.write_harvesting_config
def write_harvesting_config
Definition: cmsHarvester.py:5106


cmsHarvester.create_harvesting_config_file_name
def create_harvesting_config_file_name
Definition: cmsHarvester.py:4097











      

        

          def cmsHarvester.write_me_extraction_config 
          (
           
          self, 
        

        

          
          
           
          dataset_name 
        

        

          
          )
          
        

      




Write an ME-extraction configuration Python file.

This `ME-extraction' (ME = Monitoring Element) is the first
step of the two-step harvesting.
 

Definition at line 5139 of file cmsHarvester.py.



References create_me_summary_config_file_name(), and mergeVDriftHistosByStation.file.


 5139 


 5140     def write_me_extraction_config(self, dataset_name):


 5141         """Write an ME-extraction configuration Python file.


 5142 


 5143         This `ME-extraction' (ME = Monitoring Element) is the first


 5144         step of the two-step harvesting.


 5145 


 5146         """


 5147 


 5148         self.logger.debug("Writing ME-extraction configuration for `%s'..." % \


 5149                           dataset_name)


 5150 


 5151         # Create Python configuration.


 5152         config_contents = self.create_me_extraction_config(dataset_name)


 5153 


 5154         # Write configuration to file.


 5155         config_file_name = self. \


 5156                            create_me_summary_config_file_name(dataset_name)


 5157         try:


 5158             config_file = file(config_file_name, "w")


 5159             config_file.write(config_contents)


 5160             config_file.close()


 5161         except IOError:


 5162             self.logger.fatal("Could not write " \


 5163                               "ME-extraction configuration to file `%s'" % \


 5164                               config_file_name)


 5165             raise Error("ERROR: Could not write to file `%s'!" % \


 5166                         config_file_name)


 5167 


 5168         # End of write_me_extraction_config.





cmsHarvester.write_me_extraction_config
def write_me_extraction_config
Definition: cmsHarvester.py:5139


cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139


cmsHarvester.create_me_summary_config_file_name
def create_me_summary_config_file_name
Definition: cmsHarvester.py:4111











      

        

          def cmsHarvester.write_multicrab_config 
          (
           
          self)
          
        

      




Write a multi-CRAB job configuration Python file.
 

Definition at line 5077 of file cmsHarvester.py.



References mergeVDriftHistosByStation.file.


 5077 


 5078     def write_multicrab_config(self):


 5079         """Write a multi-CRAB job configuration Python file.


 5080 


 5081         """


 5082 


 5083         self.logger.info("Writing multi-CRAB configuration...")


 5084 


 5085         file_name_base = "multicrab.cfg"


 5086 


 5087         # Create multi-CRAB configuration.


 5088         multicrab_contents = self.create_multicrab_config()


 5089 


 5090         # Write configuration to file.


 5091         multicrab_file_name = file_name_base


 5092         try:


 5093             multicrab_file = file(multicrab_file_name, "w")


 5094             multicrab_file.write(multicrab_contents)


 5095             multicrab_file.close()


 5096         except IOError:


 5097             self.logger.fatal("Could not write " \


 5098                               "multi-CRAB configuration to file `%s'" % \


 5099                               multicrab_file_name)


 5100             raise Error("ERROR: Could not write to file `%s'!" % \


 5101                         multicrab_file_name)


 5102 


 5103         # End of write_multicrab_config.





cmsHarvester.write_multicrab_config
def write_multicrab_config
Definition: cmsHarvester.py:5077


cmsHarvester.Error
Helper class: Error exception. 
Definition: cmsHarvester.py:139







Variable Documentation






      

        

          string cmsHarvester.__author__ = "Jeroen Hegeman (jeroen.hegeman@cern.ch),"
        

      





Definition at line 38 of file cmsHarvester.py.











      

        

          string cmsHarvester.__version__ = "3.8.2p1"
        

      





File : cmsHarvest.py Authors : Jeroen Hegeman (jeroe.nosp@m.n.he.nosp@m.geman.nosp@m.@cer.nosp@m.n.ch) Niklas Pietsch (nikla.nosp@m.s.pi.nosp@m.etsch.nosp@m.@des.nosp@m.y.de) Franseco Costanza (franc.nosp@m.esco.nosp@m..cost.nosp@m.anza.nosp@m.@desy.nosp@m..de) Last change: 20100308. 


Purpose : Main program to run all kinds of harvesting. For more information please refer to the CMS Twiki url mentioned just below here. 



Definition at line 37 of file cmsHarvester.py.











      

        

          string cmsHarvester.action = "callback"
        

      





Definition at line 2055 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.all_file_names = files_info[run_number]
        

      





Definition at line 3231 of file cmsHarvester.py.











      

        

          list cmsHarvester.all_t1
        

      



Initial value:
    1 = [


    2                 "srm-cms.cern.ch",


    3                 "ccsrm.in2p3.fr",


    4                 "cmssrm-fzk.gridka.de",


    5                 "cmssrm.fnal.gov",


    6                 "gridka-dCache.fzk.de",


    7                 "srm-cms.gridpp.rl.ac.uk",


    8                 "srm.grid.sinica.edu.tw",


    9                 "srm2.grid.sinica.edu.tw",


   10                 "srmcms.pic.es",


   11                 "storm-fe-cms.cr.cnaf.infn.it"


   12                 ]




Definition at line 1722 of file cmsHarvester.py.











      

        

          cmsHarvester.caf_access
        

      





Definition at line 1105 of file cmsHarvester.py.











      

        

          cmsHarvester.callback = self.option_handler_input_Jsonrunfile,
        

      





Definition at line 2056 of file cmsHarvester.py.



Referenced by CaloDualConeSelector< HBHERecHit >.selectCallback(), CaloConeSelector< T >.selectCallback(), and edm::ESProducer.setWhatProduced().











      

        

          cmsHarvester.castor_base_dir
        

      





Definition at line 1074 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.castor_dir = self.datasets_information[dataset_name]
        

      





CRAB 




GRID 


USER 



Definition at line 4356 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.castor_path_common = self.create_castor_path_name_common(dataset_name)
        

      





DEBUG DEBUG DEBUG


This is probably only useful to make sure we don't muck


things up, right?


Figure out across how many sites this sample has been spread.




if num_sites == 1: self.logger.info("    sample is contained at a single site") else: self.logger.info("    sample is spread across %d sites" % \ num_sites) if num_sites < 1: 


NOTE: This should not happen with any valid dataset.


self.logger.warning("  --> skipping dataset which is not " \ "hosted anywhere") 


DEBUG DEBUG DEBUG end



Definition at line 5455 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.castor_paths
        

      



Initial value:
    1 = dict(zip(runs,


    2                                     [self.create_castor_path_name_special(dataset_name, i, castor_path_common) \


    3                                      for i in runs]))



archive.zip
tuple zip
Definition: archive.py:476




Definition at line 5459 of file cmsHarvester.py.











      

        

          cmsHarvester.castor_prefix = self.castor_prefix
        

      





Definition at line 4351 of file cmsHarvester.py.











      

        

          string cmsHarvester.cmd = "rfstat %s"
        

      





self.logger.debug("Path is now `%s'" % \ path) 



Definition at line 1631 of file cmsHarvester.py.











      

        

          cmsHarvester.cmssw_version = self.datasets_information[dataset_name]
        

      





Definition at line 4394 of file cmsHarvester.py.











      

        

          list cmsHarvester.complete_sites
        

      



Initial value:
    1 = [site for site in sites \


    2                                           if site in sites_with_complete_copies]




site_names_ref = set(files_info[run_number].values()[0][1]) for site_names_tmp in files_info[run_number].values()[1:]: if set(site_names_tmp[1]) != site_names_ref: mirrored = False break 



Definition at line 3276 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.config_builder = ConfigBuilder(config_options, with_input=True)
        

      





Definition at line 4794 of file cmsHarvester.py.











      

        

          string cmsHarvester.config_contents = config_builder.pythonCfgCode
        

      





In case this file is the second step (the real harvesting


step) of the two-step harvesting we have to tell it to use


our local files.




if self.harvesting_mode == "two-step": castor_dir = self.datasets_information[dataset_name] \ ["castor_path"][run] customisations.append("") customisations.append("# This is the second step (the real") customisations.append("# harvesting step) of a two-step") customisations.append("# harvesting procedure.") 


BUG BUG BUG


To be removed in production version.


customisations.append("import pdb") 


BUG BUG BUG end


customisations.append("import commands") customisations.append("import os") customisations.append("castor_dir = \"s"" % castor_dir)
            customisations.append("cmd = "rfdir s" % castor_dir")
            customisations.append("(status, output) = commands.getstatusoutput(cmd)")
            customisations.append("if status != 0:")
            customisations.append(" print "ERROR"")
            customisations.append(" raise Exception, "ERROR"")
            customisations.append("file_names = [os.path.join("rfio:s" % path, i) for i in output.split() if i.startswith("EDM_summary") and i.endswith(".root")]")
            #customisations.append("pdb.set_trace()")
            customisations.append("process.source.fileNames = cms.untracked.vstring(*file_names)")
            customisations.append("") 



Definition at line 4799 of file cmsHarvester.py.











      

        

          cmsHarvester.config_file_name = self.create_me_summary_config_file_name(dataset_name)
        

      





Only add the alarming piece to the file name if this is


a spread-out dataset.




pdb.set_trace() if self.datasets_information[dataset_name] \ ["mirrored"][run_number] == False: config_file_name = config_file_name.replace(".py", "_partial.py") 



Definition at line 4086 of file cmsHarvester.py.











      

        

          list cmsHarvester.connect_name = self.frontier_connection_name["globaltag"]
        

      





Definition at line 4833 of file cmsHarvester.py.











      

        

          dictionary cmsHarvester.country_codes
        

      



Initial value:
    1 = {


    2           "CAF" : "caf.cern.ch",


    3           "CH" : "srm-cms.cern.ch",


    4           "FR" : "ccsrm.in2p3.fr",


    5           "DE" : "cmssrm-fzk.gridka.de",


    6           "GOV" : "cmssrm.fnal.gov",


    7           "DE2" : "gridka-dCache.fzk.de",


    8           "UK" : "srm-cms.gridpp.rl.ac.uk",


    9           "TW" : "srm.grid.sinica.edu.tw",


   10           "TW2" : "srm2.grid.sinica.edu.tw",


   11           "ES" : "srmcms.pic.es",


   12           "IT" : "storm-fe-cms.cr.cnaf.infn.it"


   13             }




Definition at line 1735 of file cmsHarvester.py.











      

        

          string cmsHarvester.crab_config = "\n"
        

      





CRAB 




GRID 


USER 


CMSSW 


CAF 



Definition at line 4306 of file cmsHarvester.py.











      

        

          cmsHarvester.crab_submission
        

      





Definition at line 1132 of file cmsHarvester.py.











      

        

          list cmsHarvester.customisations = [""]
        

      





Definition at line 4829 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.dataset_name_escaped = self.escape_dataset_name(dataset_name)
        

      





Definition at line 4350 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.dataset_names = self.datasets_to_use.keys()
        

      





Definition at line 4345 of file cmsHarvester.py.











      

        

          cmsHarvester.dataset_names_after_checks = dataset_names_after_checks_tmp
        

      





Definition at line 4031 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.dataset_names_after_checks_tmp = copy.deepcopy(dataset_names_after_checks)
        

      





Definition at line 4024 of file cmsHarvester.py.











      

        

          cmsHarvester.datasets_information
        

      





Definition at line 5342 of file cmsHarvester.py.











      

        

          cmsHarvester.datasets_to_ignore
        

      





Definition at line 3458 of file cmsHarvester.py.











      

        

          cmsHarvester.datasets_to_use
        

      





Definition at line 3432 of file cmsHarvester.py.











      

        

          list cmsHarvester.datatype = self.datasets_information[dataset_name]
        

      





Definition at line 4783 of file cmsHarvester.py.











      

        

          cmsHarvester.dbs_api
        

      





Definition at line 2405 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.empty_runs = dict(tmp)
        

      





Definition at line 4008 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.es_prefer_snippet = self.create_es_prefer_snippet(dataset_name)
        

      





Definition at line 4882 of file cmsHarvester.py.











      

        

          int cmsHarvester.exit_code = 1
        

      





Definition at line 5693 of file cmsHarvester.py.











      

        

          list cmsHarvester.file_name = handler.results["file.name"]
        

      





Definition at line 3175 of file cmsHarvester.py.



Referenced by HcalLutManager.create_lut_loader(), SiPixelHistoPlotter.createStaticPlot(), SiStripHistoPlotter.createStaticPlot(), DTTPGLutFile.open(), L1TriggerLutFile.open(), TEcnaRead.ReadAverageHighFrequencyNoise(), TEcnaRead.ReadAverageLowFrequencyNoise(), TEcnaRead.ReadAverageMeanCorrelationsBetweenSamples(), TEcnaRead.ReadAveragePedestals(), TEcnaRead.ReadAverageSigmaOfCorrelationsBetweenSamples(), TEcnaRead.ReadAverageTotalNoise(), TEcnaRead.ReadCorrelationsBetweenSamples(), TEcnaRead.ReadCovariancesBetweenSamples(), TEcnaRead.ReadHighFrequencyCorrelationsBetweenChannels(), TEcnaRead.ReadHighFrequencyCovariancesBetweenChannels(), TEcnaRead.ReadHighFrequencyMeanCorrelationsBetweenStins(), TEcnaRead.ReadHighFrequencyNoise(), TEcnaRead.ReadLowFrequencyCorrelationsBetweenChannels(), TEcnaRead.ReadLowFrequencyCovariancesBetweenChannels(), TEcnaRead.ReadLowFrequencyMeanCorrelationsBetweenStins(), TEcnaRead.ReadLowFrequencyNoise(), TEcnaRead.ReadMeanCorrelationsBetweenSamples(), TEcnaRead.ReadNumberOfEventsForSamples(), TEcnaRead.ReadPedestals(), TEcnaRead.ReadRelevantCorrelationsBetweenSamples(), TEcnaRead.ReadRootFileHeader(), TEcnaRead.ReadSampleAdcValues(), TEcnaRead.ReadSampleAdcValuesSameFile(), TEcnaRead.ReadSampleMeans(), TEcnaRead.ReadSampleSigmas(), TEcnaRead.ReadSigmaOfCorrelationsBetweenSamples(), TEcnaRead.ReadStinNumbers(), TEcnaRead.ReadTotalNoise(), and TEcnaRun.WriteRootFile().











      

        

          tuple cmsHarvester.files_at_site
        

      



Initial value:
    1 = [i for (i, (j, k)) \


    2                                      in files_info[run_number].items() \


    3                                      if site_name in k]




Definition at line 3235 of file cmsHarvester.py.











      

        

          dictionary cmsHarvester.files_info = {}
        

      





Definition at line 3161 of file cmsHarvester.py.











      

        

          list cmsHarvester.files_without_sites
        

      



Initial value:
    1 = [i for (i, j) in \


    2                                    files_info[run_number].items() \


    3                                    if len(j[1]) < 1]




Definition at line 3201 of file cmsHarvester.py.











      

        

          cmsHarvester.globaltag = self.datasets_information[dataset_name]
        

      





Definition at line 4786 of file cmsHarvester.py.











      

        

          cmsHarvester.harvesting_info
        

      





Definition at line 1311 of file cmsHarvester.py.











      

        

          cmsHarvester.harvesting_mode
        

      





Definition at line 2215 of file cmsHarvester.py.











      

        

          cmsHarvester.harvesting_type
        

      





Definition at line 3859 of file cmsHarvester.py.











      

        

          string cmsHarvester.help = "Jsonfile containing dictionary of run/lumisections pairs. "
        

      





Definition at line 2053 of file cmsHarvester.py.











      

        

          string cmsHarvester.index = "site_%02d"
        

      





Definition at line 4378 of file cmsHarvester.py.



Referenced by python.listobjects._printTableInfo(), TrackingTruthAccumulator.accumulateEvent(), edm::Principal.addAliasedProduct(), ora::InputRelationalData.addBlobData(), helper::ClusterStorer.addCluster(), ora::InputRelationalData.addData(), edm::IndexIntoFile.addEntry(), ora::InputRelationalData.addId(), pat::PackedTriggerPrescales.addPrescaledTrigger(), edm::Principal.addProduct_(), DetGroupMerger.addSameLevel(), PhysicsTools::TreeReader.addTypeMulti(), PhysicsTools::TreeReader.addTypeSingle(), HcalCovarianceMatrices.addValues(), HcalCholeskyMatrices.addValues(), CastorCondObjectContainer< Item >.addValues(), HcalCondObjectContainer< Item >.addValues(), ora::InputRelationalData.addWhereId(), ora::SelectOperation.addWhereId(), edm::Principal.adjustIndexesAfterProductRegistryAddition(), AlignmentMonitorMuonResiduals.afterAlignment(), reco::Conversion.algoByName(), HcalGeometry.alignmentBarEndForIndexLocal(), HcalGeometry.alignmentOuterIndexLocal(), HcalGeometry.alignmentTransformIndexLocal(), EcalEndcapGeometry.alignmentTransformIndexLocal(), EcalBarrelGeometry.alignmentTransformIndexLocal(), JetPlotsExample< Jet >.analyze(), JetAnaPythia< Jet >.analyze(), evf::ExceptionGenerator.analyze(), EwkElecDQM.analyze(), CaloTowerAnalyzer.analyze(), EwkMuDQM.analyze(), TrackCategoriesAnalyzer.analyze(), TrackingParticleCategoriesAnalyzer.analyze(), PixelVTXMonitor.analyze(), TopElectronHLTOfflineSource.analyze(), HcalDetDiagNoiseMonitor.analyze(), SimplePhotonAnalyzer.analyze(), VertexHistoryAnalyzer.analyze(), EwkDQM.analyze(), SVTagInfoValidationAnalyzer.analyze(), TrackHistoryAnalyzer.analyze(), recoBSVTagInfoValidationAnalyzer.analyze(), HLTJetMETDQMSource.analyze(), MuonAlignmentAnalyzer.analyze(), GeneralHLTOffline.analyze(), FourVectorHLT.analyze(), HLTrigReport.analyze(), HcalRecHitsAnalyzer.analyze(), HeavyFlavorValidation.analyze(), HcalRecHitsValidation.analyze(), EcalLaserAnalyzerYousi.analyze(), DetIdSelectorTest.analyze(), OccupancyPlotter.analyze(), TriggerJSONMonitoring.analyze(), PhotonValidator.analyze(), OverlapProblemTPAnalyzer.analyze(), GenPurposeSkimmerData.analyze(), PrimaryVertexAnalyzer.analyze(), FourVectorHLTriggerOffline.analyze(), TrackerDpgAnalysis.analyze(), PrimaryVertexAnalyzer4PU.analyzeVertexCollection(), CSCConditions.anodeBXoffset(), TKinFitter.applyDeltaA(), TKinFitter.applyDeltaY(), gen::PhotosInterface.applyToVertex(), RPCConeBuilder.areConnected(), TShapeAnalysis.assignChannel(), reco::PFDisplacedVertexCandidate.associatedElements(), reco::PFBlock.associatedElements(), reco::SecondaryVertexTagInfo::TrackData.associatedToVertex(), VEcalCalibBlock.at(), attrEscape(), BackgroundHandler.BackgroundHandler(), HcalTBTiming.BeamCoincidenceHits(), RPCRecHitValid.beginRun(), RPCEfficiencySecond.beginRun(), RPCSeedPattern.BestRefRecHit(), HcalTBTiming.BH1Hits(), HcalTBTiming.BH2Hits(), HcalTBTiming.BH3Hits(), HcalTBTiming.BH4Hits(), PixelBarrelLayerBuilder.build(), TOBLayerBuilder.build(), FWCaloRecHitDigitSetProxyBuilder.build(), FWPRCaloTowerProxyBuilder.build(), FWPCaloHitProxyBuilder.build(), FWPFEcalRecHitRPProxyBuilder.build(), FWSimpleProxyBuilder.build(), FWPFEcalRecHitLegoProxyBuilder.build(), MuonSeedBuilder.build(), gen::AMPTHadronizer.build_ampt(), gen::HijingHadronizer.build_hijing(), gen::HydjetHadronizer.build_hyjet(), gen::Hydjet2Hadronizer.build_hyjet2(), SiStripFedCabling.buildFedCabling(), pos::PixelNameTranslation.buildROCsFromFEDChannel(), FWSimpleProxyBuilder.buildViewType(), evf::EvFDaqDirector.bumpFile(), LocalCacheFile.cache(), Averages.calc(), CSCTFPtLUT.calcPt(), CSCHaloAlgo.Calculate(), MedianCommonModeCalculator.calculateCommonMode(), dqmTnP::AbstractFitter.calculateEfficiency(), PileUpSubtractor.calculateOrphanInput(), npstat::ArrayND< Numeric, StackLen, StackDim >.cdfValue(), HcalDDDGeometry.cellGeomPtr(), HcalGeometry.cellGeomPtr(), FWCollectionSummaryTableManager.cellRenderer(), CSCConditions.chamberTimingCorrection(), PFPileUpAlgo.chargedHadronVertex(), PFIsolationEstimator.chargedHadronVertex(), PFPhotonIsolationCalculator.chargedHadronVertex(), RPCSeedPattern.checkSegmentAlgorithmSpecial(), RPCSeedPattern.checkSimplePattern(), TiXmlHandle.Child(), TiXmlHandle.ChildElement(), CSCConditions.chipCorrection(), FWModelContextMenuHandler.chosenItem(), TrajectorySegmentBuilder.cleanCandidates(), SeedClusterRemover.cleanup(), HLTTrackClusterRemoverNew.cleanup(), TrackClusterRemover.cleanup(), FWFromSliceSelector.clear(), MuonMillepedeAlgorithm.collect(), ora::MappingRules.columnNameForOID(), GeometricDet.component(), HFShower.compute(), HDShower.compute(), TShapeAnalysis.computeShape(), EcalUncalibRecHitRatioMethodAlgo< C >.computeTime(), EcalTPGParamBuilder.computeWeights(), L1GtVmeWriterCore.condIndex2reg(), VirtualJetProducer.copyConstituents(), MuonAlignmentFromReference.correctBField(), L1RCTParameters.correctedTPGSum(), reco::GsfComponent5D.covariance(), edm::WaitingTaskList.createNode(), CSCConditions.crossTalk(), CSCConditions.crosstalkIntercept(), CSCConditions.crosstalkSlope(), L1MuGMTPSB.CSCMuon(), CSCTFSectorProcessor.CSCTFSectorProcessor(), customizeTrackingMonitorSeedNumber.customise_trackMon_IterativeTracking_PHASE1(), customizeTrackingMonitorSeedNumber.customise_trackMon_IterativeTracking_PHASE1PU140(), customizeTrackingMonitorSeedNumber.customise_trackMon_IterativeTracking_PHASE1PU70(), pat::MuonSelector.customSelection_(), pat::ElectronSelector.customSelection_(), HLTConfigData.datasetIndex(), reco::PFDisplacedVertexCandidate.dcaPoint(), gen::EvtGenLHCInterface.decay(), evf::FastMonitoringService::Encoding.decode(), EcalShapeBase.derivative(), SurveyDet.derivatives(), SiStripLorentzAngleCalibration.derivatives(), SiPixelLorentzAngleCalibration.derivatives(), SiStripBackplaneCalibration.derivatives(), FWEventItem.destroy(), cscdqm::Detector.Detector(), HcalTopology.detId2denseIdCALIB(), HcalTopology.detId2denseIdHT(), GenericMVAJetTagComputer.discriminator(), reco::PFBlock.dist(), reco::PFDisplacedVertexCandidate.dist(), CachedTrajectory.distance(), FWCaloTowerSliceSelector.doSelect(), FWHFTowerSliceSelector.doSelect(), FWCaloTowerSliceSelector.doUnselect(), FWHFTowerSliceSelector.doUnselect(), L1MuGMTPSB.DTBXMuon(), DTROSWordType.DTROSWordType(), TKStatus.dumpTkDcsStatus(), BeamFitter.dumpTxtFile(), EcalDigiProducer.EcalDigiProducer(), EgammaTowerIsolationNew< NC >.EgammaTowerIsolationNew(), cond::IOVRange.end(), cond::IOVProxy.end(), FWModelChangeManager.endChanges(), HcalDetDiagNoiseMonitor.endLuminosityBlock(), EcalEleCalibLooper.endOfLoop(), EcalSimHitsValidProducer.energyInEBMatrix(), EcalSimHitsValidProducer.energyInEEMatrix(), EcalBarrelSimHitsValidation.energyInMatrixEB(), EcalEndcapSimHitsValidation.energyInMatrixEE(), edm::RootTree.entryNumberForIndex(), edm::RefVector< C, T, F >.erase(), edm::RefVectorBase< key_type >.eraseAtIndex(), MuonAlignmentFromReference.eraseNotSelectedResiduals(), HLTBitVariable.eval(), TrackClassifierByProxy< Collection >.evaluate(), VertexClassifierByProxy< reco::SecondaryVertexTagInfoCollection >.evaluate(), L1GtMuonCondition.evaluateCondition(), l1t::MuCondition.evaluateCondition(), L1GtCaloCondition.evaluateCondition(), l1t::CaloCondition.evaluateCondition(), PFPhotonAlgo.EvaluateGCorrMVA(), PFPhotonAlgo.EvaluateResMVA(), ora.existAttribute(), CastorQIEShape.expand(), HcalQIEShape.expand(), ClusterShapeAlgo.fast_AbsZernikeMoment(), EcalClusterToolsT< noZS >.fast_AbsZernikeMoment(), MeasurementDet.fastMeasurements(), pos::PixelCalibConfiguration.fedCardsAndChannels(), SiStripFedCabling.fedConnections(), CmsShowSearchFiles.fileEntryChanged(), VpspScanTask.fill(), PedestalsTask.fill(), ChannelPattern.Fill(), PedsFullNoiseTask.fill(), PhysicsTools::TreeReader::Value.fill(), HMassResolutionVSPart.Fill(), PFClusterShapeAlgo.fill5x5Map(), TrackingMaterialPlotter.fill_gradient(), FWHFTowerProxyBuilderBase.fillCaloData(), FWCaloTowerProxyBuilderBase.fillCaloData(), MixCollectionValidation.fillCaloHitTime(), reco.fillCovariance(), GenParticleProducer.fillDaughters(), EcalSimHitsValidProducer.fillEBMatrix(), EcalBarrelSimHitsValidation.fillEBMatrix(), EcalSimHitsValidProducer.fillEEMatrix(), EcalEndcapSimHitsValidation.fillEEMatrix(), MixCollectionValidation.fillGenParticleMulti(), EwkElecTauHistManager.fillHistograms(), EwkMuTauHistManager.fillHistograms(), Py8toJetInputHEPEVT.fillJetAlgoInput(), EcalTrigPrimFunctionalAlgo.fillMap(), JetMETHLTOfflineSource.fillMEforTriggerNTfired(), BTagHLTOfflineSource.fillMEforTriggerNTfired(), CmsShowModelPopup.fillModelPopup(), MixCollectionValidation.fillMultiplicity(), reco::Mustache.FillMustacheVar(), HcalDeadCellMonitor.fillNevents_recentdigis(), HcalDeadCellMonitor.fillNevents_recentrechits(), MuonAlignmentFromReference.fillNtuple(), FWTGeoRecoGeometry::Info.fillPoints(), FWRecoGeometryESProducer.fillPoints(), edm::IndexIntoFile.fillRunOrLumiIndexes(), MixCollectionValidation.fillSimHitTime(), TrigResRateMon.filltestHisto(), HLTRHemisphere.filter(), DYGenFilter.filter(), FWFileEntry.filterEventsWithCustomParser(), ecaldqm::LedTask.filterRunType(), find(), edm::LazyGetter< T >.find(), EcalRecHitsValidation.findBarrelMatrix(), CalibratableTest.findCandidatesInDeltaR(), TagInfoMVACategorySelector.findCategory(), FWColorRow.FindColorIndex(), EcalRecHitsValidation.findEndcapMatrix(), L1GctHardwareJetFinder.findFinalClusters(), SiStripProcessedRawDigiProducer.findInput(), TrackingMaterialAnalyser.findLayer(), CompositeTECPetal.findPar(), PFClusterShapeAlgo.findPFRHIndexFromDetId(), CalibratableTest.findPrimarySimParticles(), edm::Principal.findProductByLabel(), FFTJetPFPileupCleaner.findSomeVertexWFakes(), sistrip::MeanAndStdDev.fit(), ora::MappingRules.fkNameForIdentity(), reco::SecondaryVertexTagInfo.flightDirection(), reco::SecondaryVertexTagInfo.flightDistance(), MuonSeedBuilder.foundMatchingSegment(), tauImpactParameter::TrackHelixVertexFitter.freeParName(), npstat::ArrayND< Numeric, StackLen, StackDim >.functorFill(), CSCDBGains.gain(), CSCConditions.gain(), GammaSeries(), CSCConditions.gasGainCorrection(), gen::Hydjet2Hadronizer.generatePartonsAndHadronize(), JetTagComputer::TagInfoHelper.get(), pat::EventHypothesis.getAs(), JetTagComputer::TagInfoHelper.getBase(), SiStripDelay.getBaseDelay(), DTMuonMillepede.getbcsMatrix(), fwlite::DataGetterHelper.getBranchData(), edm::PrincipalGetAdapter.getBranchDescription(), fwlite::DataGetterHelper.getByLabel(), fwlite::DataGetterHelper.getByProductID(), edm::Principal.getByToken(), edm::PrincipalGetAdapter.getByToken_(), Gflash.getCalorimeterNumber(), DTMuonMillepede.getCcsMatrix(), CaloSubdetectorGeometry.getClosestCell(), LMFDefFabric.getColor(), GenericMVAComputerCache.getComputer(), VirtualJetProducer.getConstituents(), npstat::LinInterpolatedTableND< Numeric, Axis >.getCoords(), RxCalculator.getCRx(), edm::Principal.getExistingProduct(), SiStripPlotGain.getHisto(), SiStripCorrelateBadStripAndNoise.getHisto(), SiStripCorrelateNoise.getHisto(), Fp420AnalysisHistManager.GetHisto(), BscAnalysisHistManager.GetHisto(), Fp420AnalysisHistManager.GetHisto2(), BscAnalysisHistManager.GetHisto2(), SiStripCorrelateBadStripAndNoise.getHistos(), SiStripPlotGain.getHistos(), SiStripCorrelateNoise.getHistos(), EcalBarrelSimHitsValidation.getIdsAroundMax(), EcalEndcapSimHitsValidation.getIdsAroundMax(), ecaldqm::MESetMulti.getIndex(), SiStripDelay.getLabelName(), SiStripGain.getLabelName(), edm::Principal.getManyByType(), MuonAssociatorByHits.getMatchedIds(), TKinFitter.getMeasParticle(), SoftElectronMVAEstimator.GetMVABin(), edm::ProducerSourceBase.getNextItemType(), HcalDbOnline.getObject(), SiStripLorentzAngleCalibration.getParameter(), SiPixelLorentzAngleCalibration.getParameter(), SiStripBackplaneCalibration.getParameter(), SiStripLorentzAngleCalibration.getParameterError(), SiPixelLorentzAngleCalibration.getParameterError(), SiStripBackplaneCalibration.getParameterError(), SiStripBackplaneCalibration.getParameterForDetId(), SiPixelLorentzAngleCalibration.getParameterForDetId(), SiStripLorentzAngleCalibration.getParameterForDetId(), TkModuleGroupSelector.getParameterIndexFromDetId(), DatabasePDG.GetPDGParticleByIndex(), DatabasePDG.GetPDGParticleStatusByIndex(), pat::PackedTriggerPrescales.getPrescaleForIndex(), pat::PackedTriggerPrescales.getPrescaleForName(), edm::Principal.getProductHolder(), edm::Principal.getProductHolderByIndex(), SiStripGain.getRange(), SiStripDelay.getRcdName(), SiStripGain.getRcdName(), GBRTree2D.GetResponse(), GBRTree.GetResponse(), RxCalculator.getRFx(), RxCalculator.getROx(), RxCalculator.getRx(), CaloGeometry.getSubdetectorGeometry(), SiStripGain.getTagNorm(), SiStripDelay.getTagSign(), cond.getTotalErrors(), HcalTestNumberingScheme.getUnitID(), HcalNumberingScheme.getUnitID(), ZdcNumberingScheme.getUnitID(), CastorNumberingScheme.getUnitID(), TKinFitter.getUnmeasParticle(), CaloGeometry.getValidDetIds(), HcalCovarianceMatrices.getValues(), CastorCondObjectContainer< Item >.getValues(), HcalCholeskyMatrices.getValues(), HcalCondObjectContainer< Item >.getValues(), CachedTrajectory.getWideTrajectory(), gen::Herwig6Instance.give(), CmsShowNavigator.goToRunEvent(), TIDLayer.groupedCompatibleDetsV(), reco::GsfComponent5D.GsfComponent5D(), CastorCondObjectContainer< Item >.hashed_id(), HcalRecHitsDQMClient.HcalRecHitsEndjob(), HcalRecHitsClient.HcalRecHitsEndjob(), HFShowerFibreBundle.HFShowerFibreBundle(), HFShowerPMT.HFShowerPMT(), TEcnaHistos.HistoPlot(), HLTSummaryFilter.hltFilter(), CmsShowSearchFiles.hyperlinkClicked(), MuonSeedBuilder.IdentifyShowering(), HcalCondObjectContainerBase.indexFor(), edm::InputTag.indexFor(), HcalHPDRBXMap.indexHPD(), triggerExpression::HLTReader.init(), HcalRecHitsMaker.init(), TShapeAnalysis.init(), edm::EventProcessor.init(), module.init(), pftools::LinearCalibrator.initEijMatrix(), Thrust.initialAxis(), CombinedMVAJetTagComputer.initialize(), CSCTFSectorProcessor.initialize(), FWColorManager.initialize(), MuonAlignmentFromReference.initialize(), edm::StreamSchedule.initializeEarlyDelete(), edm::ProductRegistry.initializeLookupTables(), MultiGaussianStateTransform.innerMultiState1D(), MuonDigiCollection< CSCDetId, CSCCLCTDigi >.insertDigi(), edm::RootTree.insertEntryForIndex(), MagneticFieldGrid.interpolateAtPoint(), HLTInclusiveVBFSource.isHLTPathAccepted(), BTagHLTOfflineSource.isHLTPathAccepted(), JetMETHLTOfflineSource.isHLTPathAccepted(), CSCConditions.isInBadChamber(), btag::Matching< Delta >.isMatched1st(), lhef::Matching< Delta >.isMatched1st(), btag::Matching< Delta >.isMatched2nd(), lhef::Matching< Delta >.isMatched2nd(), RPCCosmicSeedrecHitFinder.isouterLayer(), HLTInclusiveVBFSource.isTriggerObjectFound(), BTagHLTOfflineSource.isTriggerObjectFound(), JetMETHLTOfflineSource.isTriggerObjectFound(), QuadrupletSeedMerger.isValidQuadruplet(), CSCDBGasGainCorrection.item(), CSCDBChipSpeedCorrection.item(), CSCDBGains.item(), CSCDBNoiseMatrix.item(), CSCDBPedestals.item(), CSCDBCrosstalk.item(), CSCChamberTimeCorrections.item(), gen::JetMatchingMadgraph.JetMatchingMadgraph(), gen::JetMatchingMGFastJet.JetMatchingMGFastJet(), L1AcceptBunchCrossing.L1AcceptBunchCrossing(), edm::EDConsumerBase.labelsForToken(), TrackQuality.layer(), LayerTriplets.layers(), npstat::ArrayND< Numeric, StackLen, StackDim >.linearValue(), npstat::ArrayND< Numeric, StackLen, StackDim >.linearValueAt(), CSCDBCrosstalk.linter(), python.diff_provenance.difference.list_diff(), python.diffProv.difference.list_diff(), dbUtil.dbUtil.listIndex(), CaloTPGTranscoderULUT.loadHCALCompress(), fftjetcms::FFTJetInterface.loadInputCollection(), QualityCutsAnalyzer.LoopOverJetTracksAssociation(), CSCDBCrosstalk.lslope(), HcalTBTiming.M1Hits(), HcalTBTiming.M2Hits(), HcalTBTiming.M3Hits(), main(), L1GctHardwareJetFinder.makeProtoJet(), edm.makeRefTo(), edmNew.makeRefTo(), edm.makeRefToDetSetLazyVector(), edm.makeRefToDetSetRefVector(), edm.makeRefToLazyGetter(), edm::ContainerMask< T >.mask(), pf2pat::TopProjectorAlgo< Top, Bottom >.maskAncestors(), ecaldqm::MESetMulti.MESetMulti(), FWProxyBuilderBase.modelChanges(), HLTConfigData.moduleIndex(), edm::EDConsumerBase.modulesDependentUpon(), MuonScenarioBuilder.moveCSCSectors(), MuonScenarioBuilder.moveDTSectors(), MultiGaussianStateTransform.multiState1D(), MuonGeometrySanityCheckPoint.MuonGeometrySanityCheckPoint(), HcalDDDGeometry.newCell(), HcalGeometry.newCell(), FWGUIEventDataAdder.newIndexSelected(), ora::MappingRules.newNameForSchemaObject(), cscdqm::Detector.NextAddressBoxByPartition(), CSCConditions.noiseMatrix(), HcalQIECoder.offset(), CastorQIECoder.offset(), FWDetailViewManager.openDetailViewFor(), EcalShapeBase.operator()(), Grid3D.operator()(), reco::tau::Combinatoric< T >::ValueAccessor.operator()(), edm::LazyAdapter< T >.operator()(), edm::UpdateGetterAdapter< T >.operator()(), edm::FindValue< T >.operator()(), operator<<(), edm::RefGetter< T >.operator[](), SeedingLayerSetsHits.operator[](), edm::LazyGetter< T >.operator[](), MultiGaussianStateTransform.outerMultiState1D(), TIDLayer.overlapInR(), AlignmentSurfaceDeformations.parameters(), MuonAlignmentFromReference.parseReference(), CaloTowerConstituentsMapBuilder.parseTextMap(), ParticleDecayProducer.ParticleDecayProducer(), HWWFunctions.passBaseline(), HWWFunctions.passFullLep(), CSCDBPedestals.pedestal(), CSCConditions.pedestal(), CSCDBPedestals.pedestal_rms(), CSCConditions.pedestalSigma(), PFTauMVAInputDiscriminantTranslator.PFTauMVAInputDiscriminantTranslator(), pos::PixelCalibConfiguration.PixelCalibConfiguration(), pos::PixelNameTranslation.PixelNameTranslation(), HDQMInspector.plot(), edm::PileUp.poissonDistr_OOT(), edm::PileUp.poissonDistribution(), pftools::LinearCalibrator.populateDetElIndex(), SiStripRegionCabling.position(), edm::service::Timing.postEvent(), edm::service::Timing.preEvent(), MatrixInjector.MatrixInjector.prepare(), pf2pat::TopProjectorAlgo< Top, Bottom >.printAncestors(), SiStripFedCabling.printDebug(), TShapeAnalysis.printshapeData(), SiStripFedCabling.printSummary(), ErsatzMEt.probeFinder(), PFAlgoTestBenchElectrons.processBlock(), PFAlgo.processBlock(), HcalBeamMonitor.processEvent(), TrackListCombiner.produce(), QualityFilter.produce(), JetTracksAssociationDRCalo.produce(), MuonTrackProducer.produce(), GenParticlePruner.produce(), SecondaryVertexTagInfoProxy.produce(), FastTrackMerger.produce(), cms::JetCorrectionProducer< T >.produce(), edm::LogErrorHarvester.produce(), ScalersRawToDigi.produce(), TrackCandidateProducer.produce(), pat::PATConversionProducer.produce(), HITSiStripRawToClustersRoI.produce(), pat::PATMuonProducer.produce(), cms::DigitizerFP420.produce(), pat::PATTriggerMatchEmbedder< PATObjectType >.produce(), JetDeltaRValueMapProducer< T >.produce(), pat::PATElectronProducer.produce(), SecondaryVertexProducer.produce(), reco::PhysObjectMatcher< C1, C2, S, D, Q >.produce(), ecaldqm::OccupancyClient.producePlots(), MuonDigiCollection< CSCDetId, CSCCLCTDigi >.put(), TShapeAnalysis.putAllVals(), MagneticFieldGrid.putCoordGetInd(), MagneticFieldGrid.putIndGetCoord(), SiPixelGenError.qbin(), SiStripTemplate.qbin(), SiPixelTemplate.qbin(), SiPixelTemplate.qbin_dist(), HcalDigiProducer.randomEngine(), CastorDigiProducer.randomEngine(), HcalTBDigiProducer.randomEngine(), SiStripDigitizer.randomEngine(), cms::SiPixelDigitizer.randomEngine(), edm::PileUp.randomEngine(), EcalDigiProducer.randomEngine(), EcalMixingModuleValidation.randomEngine(), L1CaloRegionDetId.rctCard(), DQMRootSource.readElements(), CSCTFSectorProcessor.readParameters(), MuonAlignmentFromReference.readTmpFiles(), edm::Principal.recombine(), fwlite::Record.Record(), edm::EDConsumerBase.recordConsumes(), GenParticlePruner.recursiveFlagDaughters(), GenParticlePruner.recursiveFlagMothers(), edm::RefGetter< T >.RefGetter(), SiStripRegionCabling.region(), python.multivaluedict.remove(), lhef::LHEEvent.removeParticle(), edmplugin::CacheParser.replaceSpaces(), BackgroundHandler.rescale(), FWColorPopup.ResetColors(), resetColors(), edm::StreamSchedule.resetEarlyDelete(), edmplugin::CacheParser.restoreSpaces(), PhotonOfflineClient.retrieveHisto(), ZeeCalibration.ringNumberCorrector(), CSCDBCrosstalk.rinter(), FWCollectionSummaryTableManager.rowHeader(), L1MuGMTPSB.RPCMuon(), CSCDBCrosstalk.rslope(), EcalTrigPrimFunctionalAlgo.run_part2(), FWEventItem.runFilter(), runInspector(), DTOccupancyTest.runOccupancyTest(), ecaldqm::LedTask.runOnDigis(), ecaldqm::LedTask.runOnPnDigis(), ecaldqm::LedTask.runOnRawData(), ecaldqm::LedTask.runOnUncalibRecHits(), PFPhotonAlgo.RunPFPhoton(), HcalTBTiming.S1Hits(), HcalTBTiming.S2Hits(), HcalTBTiming.S3Hits(), HcalTBTiming.S4Hits(), HcalDetDiagNoiseMonitor.SaveRates(), FWCaloRecHitDigitSetProxyBuilder.scaleProduct(), FWPFEcalRecHitRPProxyBuilder.scaleProduct(), reco::SecondaryVertexTagInfo.secondaryVertex(), RPCSeedPattern.SegmentAlgorithmSpecial(), FWViewContextMenuHandlerGL.select(), HLTEventSelector.select(), FWModelExpressionSelector.select(), MuonAlignmentFromReference.selectResidualsPeaks(), CmsShowSearchFiles.sendToWebBrowser(), ElectronDqmAnalyzerBase.setBookIndex(), CalibrationInterface< CategoryT, CalibDataT >.setCalibData(), Vispa.Main.Application.Application.setCurrentTabController(), FWEventItem.setDefaultDisplayProperties(), FWEventItem.setDisplayProperties(), edm::Path.setEarlyDeleteHelpers(), SimpleL1MuGMTCand.setEta(), FWPSetTableManager.setExpanded(), Particle.SetFirstDaughterIndex(), HBHEStatusBitSetter.SetFlagsFromDigi(), HBHEStatusBitSetter.SetFlagsFromRecHits(), RPCLogCone.setIdx(), Particle.SetLastDaughterIndex(), reco::PFBlock.setLink(), reco::PFDisplacedVertexCandidate.setLink(), PFElectronAlgo.SetLinks(), HcalQIECoder.setOffset(), CastorQIECoder.setOffset(), SiStripLorentzAngleCalibration.setParameter(), SiPixelLorentzAngleCalibration.setParameter(), SiStripBackplaneCalibration.setParameter(), SiStripLorentzAngleCalibration.setParameterError(), SiPixelLorentzAngleCalibration.setParameterError(), SiStripBackplaneCalibration.setParameterError(), fit::RootMinuit< Function >.setParameters(), DreamSD.setPbWO2MaterialProperties_(), SimpleL1MuGMTCand.setPhi(), reco::PFTrack.setPoint(), SimpleL1MuGMTCand.setPt(), edm::OwnArray< T, MAX_SIZE, P >.setPtr(), edm::OwnVector< T, P >.setPtr(), reco::SoftLeptonProperties.setQuality(), gen::Pythia6Service.setSLHAFromHeader(), HcalQIECoder.setSlope(), CastorQIECoder.setSlope(), CaloGeometry.setSubdetGeometry(), CaloTopology.setSubdetTopology(), FWCaloTowerDetailView.setTextInfo(), FWMuonDetailView.setTextInfo(), FWPhotonDetailView.setTextInfo(), FWTrackHitsDetailView.setTextInfo(), FWElectronDetailView.setTextInfo(), FWConvTrackHitsDetailView.setTextInfo(), edm::IndexIntoFile::IndexIntoFileItrSorted.setToLastEventInRange(), DQMRootSource.setupFile(), SiPixelPerformanceSummary.setValue(), PFElToElAssMaker.SetVars(), VertexMaker.SetVars(), PFCandidateMaker.SetVars(), MonitorElement.ShiftFillLast(), FWGUIValidatingTextEntry.showOptions(), CmsShowSearchFiles.showPrefixes(), FWModelContextMenuHandler.showSelectedModelContext(), L1DummyProducer.SimpleDigi(), SiPixelDetSummary.SiPixelDetSummary(), HcalQIECoder.slope(), CastorQIECoder.slope(), CSCChamberSpecs.specsValue(), TrackingMaterialAnalyser.split(), DQMRootOutputModule.startEndFile(), HLTConfigData.streamIndex(), edm.stripNamespace(), edm::SubProcess.SubProcess(), ReflectedIterator.subtractPedestal(), MultipleAlgoIterator.subtractPedestal(), ParametrizedSubtractor.subtractPedestal(), PileUpSubtractor.subtractPedestal(), cond::PayLoadInspector< DataT >.summary(), reco::GsfTrackExtra.tangentDeltaP(), reco::GsfTrackExtra.tangentMomentum(), reco::GsfTrackExtra.tangentPosition(), TauDQMHistPlotter.TauDQMHistPlotter(), SiPixelTemplate.temperrors(), GBRTree2D.TerminalIndex(), GBRTree.TerminalIndex(), SiStripFedCabling.terse(), TEveElementIter.TEveElementIter(), EcalShapeBase.timeIndex(), HFTimingTrust.timerr_hf(), CSCCFEBTimeSlice.timeSample(), timeshift_ns_hbheho(), timeshift_ns_hf(), reco::SecondaryVertexTagInfo.track(), reco::SecondaryVertexTagInfo.trackData(), reco::SecondaryVertexTagInfo.trackIPData(), reco::PFTrack.trajectoryPoint(), HLTConfigData.triggerIndex(), edm::TriggerNames.TriggerNames(), CSCOfflineMonitor.typeIndex(), CSCValidation.typeIndex(), uniqueElectronFinder(), TrackCategories.unknownTrack(), VertexCategories.unknownVertex(), CSCTFEvent.unpack(), edm::LazyGetter< T >.unpacked(), DCCMemBlock.unpackMemTowerData(), HcaluLUTTPGCoder.update(), FWPSetTableManager.update(), FP420Test.update(), BscTest.update(), G4StepStatistics.update(), gen::EvtGenLHCInterface.update_candlist(), EcalRegionCabling.updateEcalRefGetterWithElementIndex(), EcalRegionCabling.updateEcalRefGetterWithEtaPhi(), SeedingLayerSetsBuilder.updateEventSetup(), HcalDetDiagNoiseMonitor.UpdateHistos(), FWTableView.updateItems(), edm::EDConsumerBase.updateLookup(), SiStripRegionCabling.updateSiStripRefGetter(), HcalTBTiming.V775(), HLTMuon.validChambers(), CSCDBGasGainCorrection.value(), CSCDBChipSpeedCorrection.value(), PrimaryVertexMonitor.vertexPlots(), sistrip::EnsembleCalibrationLA.write_ensembles_text(), GctFormatTranslateMCLegacy.writeRctEmCandBlocks(), MuonAlignmentFromReference.writeTmpFiles(), and L1GtVmeWriterCore.writeVME().











      

        

          cmsHarvester.Jsonfilename
        

      





Definition at line 3708 of file cmsHarvester.py.











      

        

          cmsHarvester.Jsonlumi
        

      





Definition at line 3682 of file cmsHarvester.py.











      

        

          int cmsHarvester.loop = 0
        

      





CMSSW 




CAF 



Definition at line 4391 of file cmsHarvester.py.



Referenced by optutl::CommandLineParser._getSectionFiles(), addFiles(), RawDataConverter.ClearData(), CmsShowMainFrame.CmsShowMainFrame(), DDCheckMaterial(), SiStripTFile.dirContent(), MillePedeAlignmentAlgorithm.doIO(), LaserAlignment.DumpPosFileSet(), LaserAlignment.DumpStripFileSet(), SETSeedFinder.estimateMomentum(), PhysicsTools::MVAComputer.evalInternal(), LMFDefFabric.getColor(), LMFDefFabric.getColorFromID(), RawDataConverter.GetDigis(), LMFDefFabric.getRunTag(), LMFDefFabric.getRunTagFromID(), LMFDefFabric.getTrigType(), LMFDefFabric.getTrigTypeFromID(), pat::EventHypothesis.loop(), output(), optutl::CommandLineParser.parseArguments(), EcalDigiSelector.produce(), SiStripTFile.readDQMFormat(), TrajectoryManager.reconstruct(), stdcomb.recursive_combination(), LMFColoredTable.setColor(), LMFColoredTable.setSystem(), HcalTestAnalysis.update(), DDI::Polyhedra.volume(), and DDI::Polycone.volume().











      

        

          string cmsHarvester.marker = "\n"
        

      





Definition at line 4814 of file cmsHarvester.py.



Referenced by fireworks.addDashedArrow(), fireworks.addDashedLine(), FWPhotonLegoProxyBuilder.build(), FWMuonGlimpseProxyBuilder.build(), FWElectronLegoProxyBuilder.build(), FWElectronGlimpseProxyBuilder.build(), FWTauProxyBuilderBase.buildBaseTau(), and FWMETProxyBuilder.buildViewType().











      

        

          list cmsHarvester.marker_lines = []
        

      





Definition at line 4806 of file cmsHarvester.py.











      

        

          string cmsHarvester.metavar = "JSONRUNFILE"
        

      





Definition at line 2058 of file cmsHarvester.py.











      

        

          cmsHarvester.mirrored = None
        

      





Definition at line 3222 of file cmsHarvester.py.











      

        

          string cmsHarvester.msg = "Could not create directory `%s'"
        

      





class Handler(xml.sax.handler.ContentHandler): def startElement(self, name, attrs): if name == "result": site_name = str(attrs["STORAGEELEMENT_SENAME"]) 


TODO TODO TODO


Ugly hack to get around cases like this:


$ dbs search –query="find dataset, site, file.count where dataset=/RelValQCD_Pt_3000_3500/CMSSW_3_3_0_pre1-STARTUP31X_V4-v1/GEN-SIM-RECO"


Using DBS instance at: http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet


Processing ...




\ 


PATH STORAGEELEMENT_SENAME COUNT_FILES


_________________________________________________________________________________


/RelValQCD_Pt_3000_3500/CMSSW_3_3_0_pre1-STARTUP31X_V4-v1/GEN-SIM-RECO 1


/RelValQCD_Pt_3000_3500/CMSSW_3_3_0_pre1-STARTUP31X_V4-v1/GEN-SIM-RECO cmssrm.fnal.gov 12


/RelValQCD_Pt_3000_3500/CMSSW_3_3_0_pre1-STARTUP31X_V4-v1/GEN-SIM-RECO srm-cms.cern.ch 12


if len(site_name) < 1: return 


TODO TODO TODO end


run_number = int(attrs["RUNS_RUNNUMBER"]) file_name = str(attrs["FILES_LOGICALFILENAME"]) nevents = int(attrs["FILES_NUMBEROFEVENTS"]) 


I know, this is a bit of a kludge.


if not files_info.has_key(run_number): 


New run.


files_info[run_number] = {} files_info[run_number][file_name] = (nevents, [site_name]) elif not files_info[run_number].has_key(file_name): 


New file for a known run.


files_info[run_number][file_name] = (nevents, [site_name]) else: 


New entry for a known file for a known run.


DEBUG DEBUG DEBUG


Each file should have the same number of


events independent of the site it's at.


assert nevents == files_info[run_number][file_name][0] 


DEBUG DEBUG DEBUG end


files_info[run_number][file_name][1].append(site_name) OBSOLETE OBSOLETE OBSOLETE end 



Definition at line 1639 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.multicrab_block_name
        

      



Initial value:
    1 = self.create_multicrab_block_name( \


    2                                 dataset_name, run, index)




Definition at line 4414 of file cmsHarvester.py.











      

        

          string cmsHarvester.multicrab_config = "\n"
        

      





Definition at line 4495 of file cmsHarvester.py.











      

        

          list cmsHarvester.multicrab_config_lines = []
        

      





Definition at line 4338 of file cmsHarvester.py.











      

        

          list cmsHarvester.nevents = int(handler.results["file.numevents"][index])
        

      





Definition at line 3176 of file cmsHarvester.py.











      

        

          cmsHarvester.non_t1access
        

      





Definition at line 1089 of file cmsHarvester.py.











      

        

          cmsHarvester.nr_max_sites
        

      





Definition at line 1140 of file cmsHarvester.py.











      

        

          dictionary cmsHarvester.num_events_catalog = {}
        

      





Definition at line 3215 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.num_events_dataset = sum(tmp)
        

      





Definition at line 3986 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.num_sites
        

      



Initial value:
    1 = len(self.datasets_information[dataset_name] \


    2                                 ["sites"][run_number])




               if self.datasets_information[dataset_name]["num_events"][run_number] != 0:
                   pdb.set_trace()

 DEBUG DEBUG DEBUG end 



Definition at line 3956 of file cmsHarvester.py.











      

        

          int cmsHarvester.number_max_sites = self.nr_max_sites+1
        

      





Definition at line 4336 of file cmsHarvester.py.











      

        

          cmsHarvester.option_parser
        

      





Definition at line 1878 of file cmsHarvester.py.











      

        

          cmsHarvester.output_file_name = self.\
        

      





Definition at line 4382 of file cmsHarvester.py.



Referenced by HcalLutManager.writeLutXmlFiles().











      

        

          tuple cmsHarvester.path = os.path.join(path, piece)
        

      





else: 


Piece not in the list, fine.




                   self.logger.debug("  accepting")

 Add piece to the path we're building. self.logger.debug("!!! Skip path piece `%s'? %s" % \ (piece, str(skip_this_path_piece))) self.logger.debug("Adding piece to path...") 



Definition at line 1591 of file cmsHarvester.py.



Referenced by RPCLinkSynchroStat.add(), addFiles(), HLTPerformanceInfo.addModuleToPath(), DeDxDiscriminatorLearnerFromCalibTree.algoAnalyzeTheTree(), SiStripGainFromCalibTree.algoAnalyzeTheTree(), HLTTauDQMPathSummaryPlotter.analyze(), PixelVTXMonitor.analyze(), MuonDTDigis.analyze(), TriggerRatesMonitor.analyze(), PrintLoadingPlugins.askedToLoad(), SiStripFedCablingBuilderFromDb.assignDcuAndDetIds(), BenchmarkAnalyzer.beginJob(), JetCorrectorDBWriter.beginJob(), PFCandidateDQMAnalyzer.beginJob(), PFMETDQMAnalyzer.beginJob(), PFJetDQMAnalyzer.beginJob(), GenericBenchmarkAnalyzer.beginJob(), ProfilerService.beginPath(), MEtoEDMConverter.beginRun(), HLTHiggsSubAnalysis.beginRun(), HLTExoticaSubAnalysis.beginRun(), TriggerJSONMonitoring.beginRun(), ecaldqm::MESetEcal.book(), DQMStore.book(), HLTHiggsSubAnalysis.bookHistograms(), CSCOfflineClient.bookMonitorObject(), CSCMonitorModule.bookMonitorObject(), edmplugin::standard.cachefileName(), DQMStoreStats.calcstats(), FWGeometryTableViewBase.cdTop(), FWPSetTableManager.cellRenderer(), LStoreStorageMaker.check(), edm::Schedule.checkForCorrectness(), cleanTrailingSlashes(), ecaldqm::MESetNonObject.clone(), ecaldqm::MESetDet2D.clone(), ecaldqm::MESetProjection.clone(), ecaldqm::MESetDet0D.clone(), ecaldqm::MESetDet1D.clone(), ecaldqm::MESetTrend.clone(), ecaldqm::MESetEcal.clone(), ecaldqm::MESetMulti.clone(), ecaldqm::MESet.clone(), CmsShowSearchFiles.CmsShowSearchFiles(), cond::persistency.compareTags(), SiStripGainFromData.ComputeChargeOverPath(), edmplugin::standard.config(), convertFile(), CommissioningHistograms.copyCustomInformation(), FWTextTreeCellRenderer.coreIcondir(), FWCheckBoxIcon.coreIcondir(), coreIcondir(), SiStripCommissioningSource.createCablingTasks(), SiStripHistoId.createHistoLayer(), ecaldqm.createMESet(), evf::EvFDaqDirector.createRunOpendirMaybe(), FedRawDataInputSource.deleteFile(), do_work(), PFClient.doEfficiency(), PFClient_JetRes.doEfficiency(), MuonTestSummary.doEnergyTests(), MuonTestSummary.doKinematicsTests(), MuonTestSummary.doMuonIDTests(), PFClient.doProfiles(), PFClient.doProjection(), MuonTestSummary.doResidualsTests(), PFClient_JetRes.doSummaries(), PFClient.doSummaries(), HLTMuonValidator.dqmBeginRun(), HLTMuonOfflineAnalyzer.dqmBeginRun(), TriggerRatesMonitor.dqmBeginRun(), MuonTrackResidualsTest.dqmEndJob(), MuonRecoTest.dqmEndJob(), DTMuonMillepede.DTMuonMillepede(), DTMuonSLToSL.DTMuonSLToSL(), EcalTrivialConditionRetriever.EcalTrivialConditionRetriever(), RawEventFileWriterForBU.endOfLS(), MuonAlignmentSummary.endRun(), ESTrivialConditionRetriever.ESTrivialConditionRetriever(), CommissioningHistograms.extractHistograms(), spf::SherpackFetcher.Fetch(), SummaryGeneratorControlView.fill(), SummaryGeneratorReadoutView.fill(), SiStripCommissioningSource.fillCablingHistos(), DQMFileSaver.fillJson(), SiPixelActionExecutor.fillOccupancy(), PFDQMEventSelector.filter(), LocalFileSystem.findCachePath(), EmDQM.findEgammaPaths(), SiPixelFrameReverter.findFedId(), FWGeometry.findFile(), SiPixelFrameReverter.findLinkInFed(), SiPixelInformationExtractor.findNoisyPixels(), DQMImplNet< DQMNet::Object >.findObject(), SiPixelFrameReverter.findPixelInRoc(), SiPixelFrameReverter.findRocInDet(), SiPixelFrameReverter.findRocInLink(), RawEventFileWriterForBU.finishFileWrite(), spf::SherpackFetcher.FnFileGet(), TkHistoMap.folderDefinition(), ecaldqm::MESet.formPath(), ecaldqm::MESetEcal.generatePaths(), pos::PixelConfigFile.get(), SiPixelQuality.getBadRocPositions(), MonitorElement.getFullname(), cscdqm::HistoDef.getFullPath(), HFShowerParam.getHits(), DQMStore.getMatchingContents(), DeDxDiscriminatorProducer.GetProbability(), FedRawDataInputSource.grabNextJsonFile(), RawEventFileWriterForBU.handler(), fit::RootMinuitCommands< Function >.init(), TStorageFactoryFile.Initialize(), edm::FileInPath.initialize_(), metsig::SignAlgoResolutions.initializeJetResolutions(), DeDxDiscriminatorLearner.Learn(), edmplugin::PluginManager.load(), EcalDeadChannelRecoveryNN< DetIdT >.load_file(), edmplugin::PluginManager.loadableFor_(), FWGeometry.loadMap(), LumiCorrectionSource.LumiCorrectionSource(), main(), dqmservices::DQMFileIterator.make_path(), pos::PixelConfigFile.makeNewVersion(), TrajectoryManager.makeSinglePSimHit(), SiPixelSCurveCalibrationAnalysis.makeThresholdSummary(), MergeRootfile(), SiPixelDigitizerAlgorithm.module_killing_DB(), MonitorElement.MonitorElement(), PhysicsTools::MVAComputerESSourceBase.MVAComputerESSourceBase(), edmplugin::PluginFactoryBase.newPlugin(), RivetAnalyzer.normalizeTree(), LStoreStorageMaker.open(), StormStorageMaker.open(), StormLcgGtStorageMaker.open(), dqmservices::DQMStreamerReader.openFile_(), ora::SharedLibraryName.operator()(), SiPixelFedCablingTree.pathToDetUnit(), SiPixelFedCablingMap.pathToDetUnit(), PlotPixelMultVtxPos(), edmplugin::PluginManager.PluginManager(), DQMRootOutputModule.postForkReacquireResources(), FastTimerService.postPathEvent(), evf::FastMonitoringService.preallocate(), evf::EvFDaqDirector.preGlobalEndLumi(), evf::FastMonitoringService.preGlobalEndLumi(), FastTimerService.prePathEvent(), SiPixelActionExecutor.prephistosB(), SiPixelActionExecutor.prephistosE(), FedCablingAnalysis.print(), cond::XMLAuthenticationService::XMLAuthenticationService.processFile(), magneticfield::VolumeBasedMagneticFieldESProducer.produce(), ShallowGainCalibration.produce(), MuonSimHitProducer.produce(), RKPropagatorInS.propagateParametersOnCylinder(), RKPropagatorInS.propagateParametersOnPlane(), MEtoEDMConverter.putData(), edmplugin::CacheParser.read(), CSCGasCollisions.readCollisionTable(), DQMStore.readFilePB(), FedRawDataInputSource.readSupervisor(), DQMBasicNet.removeLocalExcept(), FedRawDataInputSource.renameToNextFree(), gen::ParameterCollector.resolve(), ecaldqm::MESetEcal.retrieve(), SiPixelDQMRocLevelAnalyzer.RocSummary(), CommissioningHistograms.runNumber(), CommissioningHistograms.runType(), CommissioningHistograms.save(), DQMStore.save(), lumi::DBConfig.setAuthentication(), cond::XMLAuthenticationService::XMLAuthenticationService.setAuthenticationPath(), SiStripCommissioningOfflineClient.setInputFiles(), SiStripHistoPlotter.setNewCondDBPlot(), SiStripHistoPlotter.setNewPlot(), SiPixelHistoPlotter.setNewPlot(), CommonAnalyzer.setPath(), PhiSymmetryCalibration_step2_SM.setUp(), PhiSymmetryCalibration_step2.setUp(), PFJetBenchmark.setup(), PFMETBenchmark.setup(), PFTauElecRejectionBenchmark.setup(), cond::CredentialStore.setUpForService(), HelixArbitraryPlaneCrossing2Order.solutionByDirection(), splitPath(), RawEventFileWriterForBU.stop(), PFJetDQMAnalyzer.storeBadEvents(), PFCandidateDQMAnalyzer.storeBadEvents(), PFMETDQMAnalyzer.storeBadEvents(), PedsOnlyAnalysis.summary(), VpspScanAnalysis.summary(), PedestalsAnalysis.summary(), NoiseAnalysis.summary(), OptoScanAnalysis.summary(), CommissioningAnalysis.summary(), PedsFullNoiseAnalysis.summary(), SiPixelFrameReverter.toCabling(), SiPixelFrameConverter.toRoc(), LumiCorrectionSource.translateFrontierConnect(), edmplugin::PluginManager.tryToLoad(), DQMStore.useQTestByMatch(), SiStripConfigDb.usingDatabase(), cond::persistency.validateTag(), cond::XMLAuthenticationService::XMLAuthenticationService.verifyFileName(), and PixelDataFormatter.word2digi().











      

        

          tuple cmsHarvester.permissions = extract_permissions(output)
        

      





Definition at line 1648 of file cmsHarvester.py.











      

        

          string cmsHarvester.permissions_new = []
        

      





Definition at line 1678 of file cmsHarvester.py.











      

        

          string cmsHarvester.permissions_target = "775"
        

      





Definition at line 1672 of file cmsHarvester.py.











      

        

          cmsHarvester.preferred_site
        

      





Definition at line 1146 of file cmsHarvester.py.











      

        

          cmsHarvester.ref_hist_mappings_file_name
        

      





Definition at line 2257 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.run_number = int(handler.results["run.number"][index])
        

      





Definition at line 3174 of file cmsHarvester.py.



Referenced by HcalDetDiagNoiseMonitor.analyze(), BlockFormatter.DigiToRaw(), TEcnaRead.FileParameters(), HcalChannelQualityXml.HcalChannelQualityXml(), HcalDetDiagNoiseMonitor.HcalDetDiagNoiseMonitor(), HcalL1TriggerObjectsXml.HcalL1TriggerObjectsXml(), TEcnaHeader.HeaderParameters(), HcalChannelDataXml.init_data(), HcalDetDiagNoiseMonitor.SaveRates(), HcalChannelDataXml.set_header_run_number(), SiStripCommissioningOfflineClient.setInputFiles(), and XMLHTRZeroSuppressionLoader.XMLHTRZeroSuppressionLoader().











      

        

          list cmsHarvester.runs = self.datasets_to_use[dataset_name]
        

      





Definition at line 4349 of file cmsHarvester.py.











      

        

          cmsHarvester.runs_to_ignore
        

      





Definition at line 3555 of file cmsHarvester.py.











      

        

          cmsHarvester.runs_to_use
        

      





Definition at line 3531 of file cmsHarvester.py.











      

        

          cmsHarvester.saveByLumiSection
        

      





Definition at line 1119 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.se_name = choice(t1_sites)
        

      





Definition at line 1791 of file cmsHarvester.py.











      

        

          string cmsHarvester.sep = "#"
        

      





Definition at line 4807 of file cmsHarvester.py.



Referenced by HLTMonBitSummary.analyze(), PrimaryVertexAnalyzer4PU.analyzeVertexCollection(), PrimaryVertexAnalyzer4PU.analyzeVertexCollectionTP(), edm::Entry.Entry(), ExpressionVariable< Object, label >.ExpressionVariable(), LumiProducer.fillRunCache(), ElectronDqmAnalyzerBase.find(), fwlite::DataGetterHelper.getBranchDataFor(), RegressionHelper.getEcalRegression(), HLTPixlMBForAlignmentFilter.hltFilter(), fit::RootMinuitCommands< Function >.init(), Tokenizer.join(), std.operator<<(), HcalTopologyRestrictionParser.parse(), lumi::NormDML.parseAfterglows(), ParticleReplacerZtautau.ParticleReplacerZtautau(), ClhepEvaluator.prepare(), ElectronIDValueMapProducer.produce(), pat::PATPhotonProducer.produce(), ElectronEnergyRegressionEvaluate.regressionUncertaintyNoTrkVar(), ElectronEnergyRegressionEvaluate.regressionUncertaintyNoTrkVarV1(), ElectronEnergyRegressionEvaluate.regressionUncertaintyWithSubClusters(), ElectronEnergyRegressionEvaluate.regressionUncertaintyWithTrkVar(), ElectronEnergyRegressionEvaluate.regressionUncertaintyWithTrkVarV1(), ElectronEnergyRegressionEvaluate.regressionUncertaintyWithTrkVarV2(), ElectronEnergyRegressionEvaluate.regressionValueNoTrkVar(), ElectronEnergyRegressionEvaluate.regressionValueNoTrkVarV1(), ElectronEnergyRegressionEvaluate.regressionValueWithSubClusters(), ElectronEnergyRegressionEvaluate.regressionValueWithTrkVar(), ElectronEnergyRegressionEvaluate.regressionValueWithTrkVarV1(), ElectronEnergyRegressionEvaluate.regressionValueWithTrkVarV2(), BaselinePFSCRegression.set(), cond::UpdateStamp.stamp(), edm.tokenize(), and cond::IOVSequence.updateMetadata().











      

        

          list cmsHarvester.site_name = None
        

      





Definition at line 1772 of file cmsHarvester.py.











      

        

          list cmsHarvester.site_names = list(set([j for i in files_info[run_number].values() for j in i[1]]))
        

      





Definition at line 3217 of file cmsHarvester.py.











      

        

          list cmsHarvester.sites = [self.preferred_site]
        

      





Definition at line 1761 of file cmsHarvester.py.



Referenced by edm::service::SiteLocalConfigService.parse().











      

        

          list cmsHarvester.sites_forbidden = []
        

      





Definition at line 1708 of file cmsHarvester.py.











      

        

          list cmsHarvester.sites_with_complete_copies = []
        

      





Definition at line 3233 of file cmsHarvester.py.











      

        

          cmsHarvester.skip_this_path_piece = True
        

      





self.logger.debug("Checking CASTOR path piece `%s'" % \ piece) 


self.logger.debug("Checking `%s' against `%s'" % \ (castor_path_pieces[piece_index + check_size], castor_paths_dont_touch[check_size])) self.logger.debug("  skipping") 



Definition at line 1583 of file cmsHarvester.py.











      

        

          list cmsHarvester.t1_sites = []
        

      





Definition at line 1778 of file cmsHarvester.py.











      

        

          dictionary cmsHarvester.tmp
        

      



Initial value:
    1 = [j for (i, j) in self.datasets_information \


    2                    [dataset_name]["num_events"].items() \


    3                    if i in self.datasets_to_use[dataset_name]]




TODO TODO TODO


Need to think about where this should go, but


somewhere we have to move over the fact that we want


to process all runs for each dataset that we're


considering.




This basically means copying over the 


information from self.datasets_information[]["runs"]


to self.datasets_to_use[].


for dataset_name in self.datasets_to_use.keys(): self.datasets_to_use[dataset_name] = self.datasets_information[dataset_name]["runs"] 


TODO TODO TODO end


OBSOLETE OBSOLETE OBSOLETE end tmp = self.datasets_information[dataset_name] \ ["num_events"] 



Definition at line 3983 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.traceback_string = traceback.format_exc()
        

      





Definition at line 5718 of file cmsHarvester.py.











      

        

          string cmsHarvester.twiki_url = "https://twiki.cern.ch/twiki/bin/view/CMS/CmsHarvester"
        

      





Definition at line 41 of file cmsHarvester.py.











      

        

          string cmsHarvester.type = "string"
        

      





Definition at line 2057 of file cmsHarvester.py.











      

        

          tuple cmsHarvester.use_es_prefer = (self.harvesting_type == "RelVal")
        

      





Definition at line 4858 of file cmsHarvester.py.











      

        

          cmsHarvester.use_refs = use_es_preferor\
        

      





Definition at line 4859 of file cmsHarvester.py.











      

        

          cmsHarvester.UserName = output
        

      





Definition at line 4331 of file cmsHarvester.py.











      

        

          cmsHarvester.workflow_name = dataset_name
        

      





Definition at line 4887 of file cmsHarvester.py.









Generated for CMSSW Reference Manual by doxygen 1.8.5