CMS 3D CMS Logo

Classes | Namespaces | Functions | Variables
cmsHarvester.py File Reference

Go to the source code of this file.

Classes

class  cmsHarvester.CMSHarvester
 CMSHarvester class. More...
 
class  cmsHarvester.CMSHarvesterHelpFormatter
 Helper class: CMSHarvesterHelpFormatter. More...
 
class  cmsHarvester.DBSXMLHandler
 Helper class: DBSXMLHandler. More...
 
class  cmsHarvester.Error
 Helper class: Error exception. More...
 
class  cmsHarvester.Usage
 Helper class: Usage exception. More...
 

Namespaces

 cmsHarvester
 

Functions

def cmsHarvester.build_dataset_ignore_list (self)
 
def cmsHarvester.build_dataset_list (self, input_method, input_name)
 class Handler(xml.sax.handler.ContentHandler): def startElement(self, name, attrs): if name == "result": site_name = str(attrs["STORAGEELEMENT_SENAME"])

TODO TODO TODO

Ugly hack to get around cases like this:

$ dbs search –query="find dataset, site, file.count where dataset=/RelValQCD_Pt_3000_3500/CMSSW_3_3_0_pre1-STARTUP31X_V4-v1/GEN-SIM-RECO"

Using DBS instance at: http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet

Processing ...

More...
 
def cmsHarvester.build_dataset_use_list (self)
 
def cmsHarvester.build_datasets_information (self)
 
def cmsHarvester.build_runs_ignore_list (self)
 
def cmsHarvester.build_runs_list (self, input_method, input_name)
 
def cmsHarvester.build_runs_use_list (self)
 
def cmsHarvester.check_cmssw (self)
 
def cmsHarvester.check_dataset_list (self)
 
def cmsHarvester.check_dbs (self)
 
def cmsHarvester.check_globaltag (self, globaltag=None)
 
def cmsHarvester.check_globaltag_contains_ref_hist_key (self, globaltag, connect_name)
 
def cmsHarvester.check_globaltag_exists (self, globaltag, connect_name)
 
def cmsHarvester.check_input_status (self)
 
def cmsHarvester.check_ref_hist_mappings (self)
 
def cmsHarvester.check_ref_hist_tag (self, tag_name)
 
def cmsHarvester.create_and_check_castor_dir (self, castor_dir)
 
def cmsHarvester.create_and_check_castor_dirs (self)
 
def cmsHarvester.create_castor_path_name_common (self, dataset_name)
 
def cmsHarvester.create_castor_path_name_special (self, dataset_name, run_number, castor_path_common)
 
def cmsHarvester.create_config_file_name (self, dataset_name, run_number)
 
def cmsHarvester.create_crab_config (self)
 
def cmsHarvester.create_es_prefer_snippet (self, dataset_name)
 
def cmsHarvester.create_harvesting_config (self, dataset_name)
 
def cmsHarvester.create_harvesting_config_file_name (self, dataset_name)
 

Only add the alarming piece to the file name if this is

a spread-out dataset.

More...
 
def cmsHarvester.create_harvesting_output_file_name (self, dataset_name, run_number)
 
def cmsHarvester.create_me_extraction_config (self, dataset_name)
 

In case this file is the second step (the real harvesting

step) of the two-step harvesting we have to tell it to use

our local files.

More...
 
def cmsHarvester.create_me_summary_config_file_name (self, dataset_name)
 
def cmsHarvester.create_me_summary_output_file_name (self, dataset_name)
 
def cmsHarvester.create_multicrab_block_name (self, dataset_name, run_number, index)
 
def cmsHarvester.create_multicrab_config (self)
 
def cmsHarvester.create_output_file_name (self, dataset_name, run_number=None)
 
def cmsHarvester.dbs_check_dataset_spread (self, dataset_name)
 def dbs_resolve_dataset_number_of_sites(self, dataset_name): """Ask DBS across how many sites this dataset has been spread out. More...
 
def cmsHarvester.dbs_resolve_cmssw_version (self, dataset_name)
 
def cmsHarvester.dbs_resolve_dataset_name (self, dataset_name)
 
def cmsHarvester.dbs_resolve_datatype (self, dataset_name)
 
def cmsHarvester.dbs_resolve_globaltag (self, dataset_name)
 
def cmsHarvester.dbs_resolve_number_of_events (self, dataset_name, run_number=None)
 
def cmsHarvester.dbs_resolve_runs (self, dataset_name)
 def dbs_resolve_dataset_number_of_events(self, dataset_name): """Ask DBS across how many events this dataset has been spread out. More...
 
def cmsHarvester.escape_dataset_name (self, dataset_name)
 if self.datasets_information[dataset_name]["num_events"][run_number] != 0: pdb.set_trace() DEBUG DEBUG DEBUG end More...
 
def cmsHarvester.load_ref_hist_mappings (self)
 
def cmsHarvester.option_handler_caf_access (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_castor_dir (self, option, opt_str, value, parser)
 def option_handler_dataset_name(self, option, opt_str, value, parser): """Specify the name(s) of the dataset(s) to be processed. More...
 
def cmsHarvester.option_handler_crab_submission (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_list_types (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_no_t1access (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_preferred_site (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_saveByLumiSection (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_sites (self, option, opt_str, value, parser)
 
def cmsHarvester.parse_cmd_line_options (self)
 
def cmsHarvester.pick_a_site (self, sites, cmssw_version)
 self.logger.debug("Checking CASTOR path piece `%s'" % \ piece) More...
 
def cmsHarvester.process_dataset_ignore_list (self)
 
def cmsHarvester.process_runs_use_and_ignore_lists (self)
 
def cmsHarvester.ref_hist_mappings_needed (self, dataset_name=None)
 
def cmsHarvester.run (self)
 
def cmsHarvester.setup_dbs (self)
 

Now we try to do a very simple DBS search.

More...
 
def cmsHarvester.setup_harvesting_info (self)
 
def cmsHarvester.show_exit_message (self)
 

DEBUG DEBUG DEBUG

This is probably only useful to make sure we don't muck

things up, right?

Figure out across how many sites this sample has been spread.

More...
 
def cmsHarvester.singlify_datasets (self)
 
def cmsHarvester.write_crab_config (self)
 def create_harvesting_config(self, dataset_name): """Create the Python harvesting configuration for a given job. More...
 
def cmsHarvester.write_harvesting_config (self, dataset_name)
 
def cmsHarvester.write_me_extraction_config (self, dataset_name)
 
def cmsHarvester.write_multicrab_config (self)
 

Variables

 cmsHarvester.all_sites_found
 

CAF

More...
 
 cmsHarvester.caf_access
 
 cmsHarvester.castor_base_dir
 
 cmsHarvester.cmd_line_opts
 
 cmsHarvester.cmssw_version
 
 cmsHarvester.crab_submission
 
 cmsHarvester.datasets_information
 
 cmsHarvester.datasets_to_ignore
 
 cmsHarvester.datasets_to_use
 
 cmsHarvester.dbs_api
 
 cmsHarvester.globaltag
 
 cmsHarvester.harvesting_info
 
 cmsHarvester.harvesting_mode
 
 cmsHarvester.harvesting_type
 
 cmsHarvester.Jsonfilename
 
 cmsHarvester.Jsonlumi
 

CRAB

More...
 
 cmsHarvester.non_t1access
 
 cmsHarvester.nr_max_sites
 
 cmsHarvester.option_parser
 
 cmsHarvester.preferred_site
 
 cmsHarvester.ref_hist_mappings_file_name
 
 cmsHarvester.runs_to_ignore
 
 cmsHarvester.runs_to_use
 
 cmsHarvester.saveByLumiSection