CMS 3D CMS Logo

Classes | Namespaces | Functions | Variables
cmsHarvester.py File Reference

Go to the source code of this file.

Classes

class  cmsHarvester.CMSHarvester
 CMSHarvester class. More...
 
class  cmsHarvester.CMSHarvesterHelpFormatter
 Helper class: CMSHarvesterHelpFormatter. More...
 
class  cmsHarvester.DBSXMLHandler
 Helper class: DBSXMLHandler. More...
 
class  cmsHarvester.Error
 Helper class: Error exception. More...
 
class  cmsHarvester.Usage
 Helper class: Usage exception. More...
 

Namespaces

 cmsHarvester
 

Functions

def cmsHarvester.build_dataset_ignore_list (self)
 
def cmsHarvester.build_dataset_list (self, input_method, input_name)
 
def cmsHarvester.build_dataset_use_list (self)
 
def cmsHarvester.build_datasets_information (self)
 
def cmsHarvester.build_runs_ignore_list (self)
 
def cmsHarvester.build_runs_list (self, input_method, input_name)
 
def cmsHarvester.build_runs_use_list (self)
 
def cmsHarvester.check_cmssw (self)
 
def cmsHarvester.check_dataset_list (self)
 
def cmsHarvester.check_dbs (self)
 
def cmsHarvester.check_globaltag (self, globaltag=None)
 
def cmsHarvester.check_globaltag_contains_ref_hist_key (self, globaltag, connect_name)
 
def cmsHarvester.check_globaltag_exists (self, globaltag, connect_name)
 
def cmsHarvester.check_input_status (self)
 
def cmsHarvester.check_ref_hist_mappings (self)
 
def cmsHarvester.check_ref_hist_tag (self, tag_name)
 
def cmsHarvester.create_and_check_castor_dir (self, castor_dir)
 
def cmsHarvester.create_and_check_castor_dirs (self)
 
def cmsHarvester.create_castor_path_name_common (self, dataset_name)
 
def cmsHarvester.create_castor_path_name_special (self, dataset_name, run_number, castor_path_common)
 
def cmsHarvester.create_config_file_name (self, dataset_name, run_number)
 
def cmsHarvester.create_crab_config (self)
 
def cmsHarvester.create_harvesting_config (self, dataset_name)
 
def cmsHarvester.create_harvesting_config_file_name (self, dataset_name)
 
def cmsHarvester.create_harvesting_output_file_name (self, dataset_name, run_number)
 
def cmsHarvester.create_me_extraction_config (self, dataset_name)
 
def cmsHarvester.create_me_summary_config_file_name (self, dataset_name)
 
def cmsHarvester.create_me_summary_output_file_name (self, dataset_name)
 
def cmsHarvester.create_multicrab_block_name (self, dataset_name, run_number, index)
 
def cmsHarvester.create_multicrab_config (self)
 
def cmsHarvester.create_output_file_name (self, dataset_name, run_number=None)
 
def cmsHarvester.dbs_check_dataset_spread (self, dataset_name)
 
def cmsHarvester.dbs_resolve_cmssw_version (self, dataset_name)
 
def cmsHarvester.dbs_resolve_dataset_name (self, dataset_name)
 
def cmsHarvester.dbs_resolve_datatype (self, dataset_name)
 
def cmsHarvester.dbs_resolve_globaltag (self, dataset_name)
 
def cmsHarvester.dbs_resolve_number_of_events (self, dataset_name, run_number=None)
 
def cmsHarvester.dbs_resolve_runs (self, dataset_name)
 
def cmsHarvester.escape_dataset_name (self, dataset_name)
 
def cmsHarvester.load_ref_hist_mappings (self)
 
def cmsHarvester.option_handler_caf_access (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_castor_dir (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_crab_submission (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_list_types (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_no_t1access (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_preferred_site (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_saveByLumiSection (self, option, opt_str, value, parser)
 
def cmsHarvester.option_handler_sites (self, option, opt_str, value, parser)
 
def cmsHarvester.parse_cmd_line_options (self)
 
def cmsHarvester.pick_a_site (self, sites, cmssw_version)
 
def cmsHarvester.process_dataset_ignore_list (self)
 
def cmsHarvester.process_runs_use_and_ignore_lists (self)
 
def cmsHarvester.ref_hist_mappings_needed (self, dataset_name=None)
 
def cmsHarvester.run (self)
 
def cmsHarvester.setup_dbs (self)
 
def cmsHarvester.setup_harvesting_info (self)
 
def cmsHarvester.show_exit_message (self)
 
def cmsHarvester.singlify_datasets (self)
 
def cmsHarvester.write_crab_config (self)
 
def cmsHarvester.write_harvesting_config (self, dataset_name)
 
def cmsHarvester.write_me_extraction_config (self, dataset_name)
 
def cmsHarvester.write_multicrab_config (self)
 

Variables

 cmsHarvester.__author__
 
 cmsHarvester.__version__
 
 cmsHarvester.all_file_names
 
 cmsHarvester.all_sites_found
 

CAF

More...
 
 cmsHarvester.caf_access
 
 cmsHarvester.castor_base_dir
 
 cmsHarvester.castor_path_checks_cache
 self.logger.debug("Path is now `%s'" % \ path) More...
 
 cmsHarvester.castor_path_common
 

DEBUG DEBUG DEBUG

This is probably only useful to make sure we don't muck

things up, right?

Figure out across how many sites this sample has been spread.

More...
 
 cmsHarvester.castor_paths
 
 cmsHarvester.cmd
 
 cmsHarvester.cmd_line_opts
 
 cmsHarvester.cmssw_version
 
 cmsHarvester.complete_sites
 site_names_ref = set(files_info[run_number].values()[0][1]) for site_names_tmp in files_info[run_number].values()[1:]: if set(site_names_tmp[1]) != site_names_ref: mirrored = False break More...
 
 cmsHarvester.config_contents
 

In case this file is the second step (the real harvesting

step) of the two-step harvesting we have to tell it to use

our local files.

More...
 
 cmsHarvester.config_file_name
 

Only add the alarming piece to the file name if this is

a spread-out dataset.

More...
 
 cmsHarvester.crab_submission
 
 cmsHarvester.dataset_names_after_checks
 
 cmsHarvester.dataset_names_after_checks_tmp
 
 cmsHarvester.datasets_information
 
 cmsHarvester.datasets_to_ignore
 
 cmsHarvester.datasets_to_use
 
 cmsHarvester.dbs_api
 
 cmsHarvester.empty_runs
 
 cmsHarvester.exit_code
 
 cmsHarvester.file_name
 
 cmsHarvester.files_at_site
 
 cmsHarvester.files_info
 
 cmsHarvester.files_without_sites
 
 cmsHarvester.globaltag
 
 cmsHarvester.harvesting_info
 
 cmsHarvester.harvesting_mode
 
 cmsHarvester.harvesting_type
 
 cmsHarvester.Jsonfilename
 
 cmsHarvester.Jsonlumi
 

CRAB

More...
 
 cmsHarvester.mirrored
 
 cmsHarvester.msg
 class Handler(xml.sax.handler.ContentHandler): def startElement(self, name, attrs): if name == "result": site_name = str(attrs["STORAGEELEMENT_SENAME"])

TODO TODO TODO

Ugly hack to get around cases like this:

$ dbs search –query="find dataset, site, file.count where dataset=/RelValQCD_Pt_3000_3500/CMSSW_3_3_0_pre1-STARTUP31X_V4-v1/GEN-SIM-RECO"

Using DBS instance at: http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet

Processing ...

More...
 
 cmsHarvester.nevents
 
 cmsHarvester.non_t1access
 
 cmsHarvester.nr_max_sites
 
 cmsHarvester.num_events_catalog
 
 cmsHarvester.num_events_dataset
 
 cmsHarvester.num_sites
 if self.datasets_information[dataset_name]["num_events"][run_number] != 0: pdb.set_trace() DEBUG DEBUG DEBUG end More...
 
 cmsHarvester.option_parser
 
 cmsHarvester.output
 
 cmsHarvester.path
 else:

Piece not in the list, fine.

More...
 
 cmsHarvester.permissions
 
 cmsHarvester.permissions_new
 
 cmsHarvester.permissions_target
 
 cmsHarvester.preferred_site
 
 cmsHarvester.ref_hist_mappings_file_name
 
 cmsHarvester.run_number
 
 cmsHarvester.runs_to_ignore
 
 cmsHarvester.runs_to_use
 
 cmsHarvester.saveByLumiSection
 
 cmsHarvester.site_names
 
 cmsHarvester.sites_with_complete_copies
 
 cmsHarvester.skip_this_path_piece
 self.logger.debug("Checking CASTOR path piece `%s'" % \ piece) More...
 
 cmsHarvester.status
 
 cmsHarvester.tmp
 

TODO TODO TODO

Need to think about where this should go, but

somewhere we have to move over the fact that we want

to process all runs for each dataset that we're

considering.

More...
 
 cmsHarvester.traceback_string
 
 cmsHarvester.twiki_url