Public Member Functions
def	__init__ (self, markup, overrideEncodings=[], smartQuotesTo='xml', isHTML=False)

def	find_codec (self, charset)

Public Attributes
	declaredHTMLEncoding

	markup

	originalEncoding

	smartQuotesTo

	triedEncodings

	unicode

Private Member Functions
def	_codec (self, charset)

def	_convertFrom (self, proposed)

def	_detectEncoding (self, xml_data, isHTML=False)

def	_ebcdic_to_ascii (self, s)

def	_subMSChar (self, orig)

def	_toUnicode (self, data, encoding)

Detailed Description

A class for detecting the encoding of a *ML document and
converting it to a Unicode string. If the source encoding is
windows-1252, can replace MS smart quotes with their HTML or XML
equivalents.

Definition at line 1757 of file BeautifulSoup.py.

Constructor & Destructor Documentation

def BeautifulSoup.UnicodeDammit.__init__	(	self,
		markup,
		overrideEncodings = `[]`,
		smartQuotesTo = `'xml'`,
		isHTML = `False`
	)

Definition at line 1771 of file BeautifulSoup.py.

                  smartQuotesTo='xml', isHTML=False):
         self.declaredHTMLEncoding = None
         self.markup, documentEncoding, sniffedEncoding = \
                      self._detectEncoding(markup, isHTML)
         self.smartQuotesTo = smartQuotesTo
         self.triedEncodings = []
         if markup == '' or isinstance(markup, unicode):
             self.originalEncoding = None
             self.unicode = unicode(markup)
             return
 
         u = None
         for proposedEncoding in overrideEncodings:
             u = self._convertFrom(proposedEncoding)
             if u: break
         if not u:
             for proposedEncoding in (documentEncoding, sniffedEncoding):
                 u = self._convertFrom(proposedEncoding)
                 if u: break
 
         # If no luck and we have auto-detection library, try that:
         if not u and chardet and not isinstance(self.markup, unicode):
             u = self._convertFrom(chardet.detect(self.markup)['encoding'])
 
         # As a last resort, try utf-8 and windows-1252:
         if not u:
             for proposed_encoding in ("utf-8", "windows-1252"):
                 u = self._convertFrom(proposed_encoding)
                 if u: break
 
         self.unicode = u
         if not u: self.originalEncoding = None
 

Member Function Documentation

def BeautifulSoup.UnicodeDammit._codec	(	self,
		charset
	)

private

Definition at line 1942 of file BeautifulSoup.py.

Referenced by BeautifulSoup.UnicodeDammit.find_codec().

     def _codec(self, charset):
         if not charset: return charset
         codec = None
         try:
             codecs.lookup(charset)
             codec = charset
         except (LookupError, ValueError):
             pass
         return codec
 

def BeautifulSoup.UnicodeDammit._convertFrom	(	self,
		proposed
	)

private

Definition at line 1815 of file BeautifulSoup.py.

References BeautifulSoup.UnicodeDammit._subMSChar(), BeautifulSoup.UnicodeDammit._toUnicode(), BeautifulSoup.UnicodeDammit.find_codec(), recoMuon.in, BeautifulSoup.BeautifulStoneSoup.markup, BeautifulSoup.UnicodeDammit.markup, BeautifulSoup.BeautifulStoneSoup.smartQuotesTo, BeautifulSoup.UnicodeDammit.smartQuotesTo, and BeautifulSoup.UnicodeDammit.triedEncodings.

     def _convertFrom(self, proposed):
         proposed = self.find_codec(proposed)
         if not proposed or proposed in self.triedEncodings:
             return None
         self.triedEncodings.append(proposed)
         markup = self.markup
 
         # Convert smart quotes to HTML if coming from an encoding
         # that might have them.
         if self.smartQuotesTo and proposed.lower() in("windows-1252",
                                                       "iso-8859-1",
                                                       "iso-8859-2"):
             markup = re.compile("([\x80-\x9f])").sub \
                      (lambda x: self._subMSChar(x.group(1)),
                       markup)
 
         try:
             # print "Trying to convert document to %s" % proposed
             u = self._toUnicode(markup, proposed)
             self.markup = u
             self.originalEncoding = proposed
         except Exception as e:
             # print "That didn't work!"
             # print e
             return None
         #print "Correct encoding: %s" % proposed
         return self.markup
 

def BeautifulSoup.UnicodeDammit._detectEncoding	(	self,
		xml_data,
		isHTML = `False`
	)

private

Given a document, tries to detect its XML encoding.

Definition at line 1868 of file BeautifulSoup.py.

References BeautifulSoup.UnicodeDammit._ebcdic_to_ascii(), BeautifulSoup.BeautifulStoneSoup.declaredHTMLEncoding, BeautifulSoup.BeautifulSoup.declaredHTMLEncoding, BeautifulSoup.UnicodeDammit.declaredHTMLEncoding, alcaDQMUpload.encode(), match(), and BeautifulSoup.UnicodeDammit.unicode.

     def _detectEncoding(self, xml_data, isHTML=False):
         """Given a document, tries to detect its XML encoding."""
         xml_encoding = sniffed_xml_encoding = None
         try:
             if xml_data[:4] == '\x4c\x6f\xa7\x94':
                 # EBCDIC
                 xml_data = self._ebcdic_to_ascii(xml_data)
             elif xml_data[:4] == '\x00\x3c\x00\x3f':
                 # UTF-16BE
                 sniffed_xml_encoding = 'utf-16be'
                 xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
             elif (len(xml_data) >= 4) and (xml_data[:2] == '\xfe\xff') \
                      and (xml_data[2:4] != '\x00\x00'):
                 # UTF-16BE with BOM
                 sniffed_xml_encoding = 'utf-16be'
                 xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
             elif xml_data[:4] == '\x3c\x00\x3f\x00':
                 # UTF-16LE
                 sniffed_xml_encoding = 'utf-16le'
                 xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
             elif (len(xml_data) >= 4) and (xml_data[:2] == '\xff\xfe') and \
                      (xml_data[2:4] != '\x00\x00'):
                 # UTF-16LE with BOM
                 sniffed_xml_encoding = 'utf-16le'
                 xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
             elif xml_data[:4] == '\x00\x00\x00\x3c':
                 # UTF-32BE
                 sniffed_xml_encoding = 'utf-32be'
                 xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
             elif xml_data[:4] == '\x3c\x00\x00\x00':
                 # UTF-32LE
                 sniffed_xml_encoding = 'utf-32le'
                 xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
             elif xml_data[:4] == '\x00\x00\xfe\xff':
                 # UTF-32BE with BOM
                 sniffed_xml_encoding = 'utf-32be'
                 xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
             elif xml_data[:4] == '\xff\xfe\x00\x00':
                 # UTF-32LE with BOM
                 sniffed_xml_encoding = 'utf-32le'
                 xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
             elif xml_data[:3] == '\xef\xbb\xbf':
                 # UTF-8 with BOM
                 sniffed_xml_encoding = 'utf-8'
                 xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
             else:
                 sniffed_xml_encoding = 'ascii'
                 pass
         except:
             xml_encoding_match = None
         xml_encoding_match = re.compile(
             '^<\?.*encoding=[\'"](.*?)[\'"].*\?>').match(xml_data)
         if not xml_encoding_match and isHTML:
             regexp = re.compile('<\s*meta[^>]+charset=([^>]*?)[;\'">]', re.I)
             xml_encoding_match = regexp.search(xml_data)
         if xml_encoding_match is not None:
             xml_encoding = xml_encoding_match.groups()[0].lower()
             if isHTML:
                 self.declaredHTMLEncoding = xml_encoding
             if sniffed_xml_encoding and \
                (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
                                  'iso-10646-ucs-4', 'ucs-4', 'csucs4',
                                  'utf-16', 'utf-32', 'utf_16', 'utf_32',
                                  'utf16', 'u16')):
                 xml_encoding = sniffed_xml_encoding
         return xml_data, xml_encoding, sniffed_xml_encoding
 
 

def BeautifulSoup.UnicodeDammit._ebcdic_to_ascii	(	self,
		s
	)

private

Definition at line 1953 of file BeautifulSoup.py.

References __class__< T >.__class__(), pat::__class__.__class__(), join(), genParticles_cff.map, and edm.print().

Referenced by BeautifulSoup.UnicodeDammit._detectEncoding().

     def _ebcdic_to_ascii(self, s):
         c = self.__class__
         if not c.EBCDIC_TO_ASCII_MAP:
             emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
                     16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
                     128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
                     144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
                     32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
                     38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
                     45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
                     186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
                     195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,
                     201,202,106,107,108,109,110,111,112,113,114,203,204,205,
                     206,207,208,209,126,115,116,117,118,119,120,121,122,210,
                     211,212,213,214,215,216,217,218,219,220,221,222,223,224,
                     225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72,
                     73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81,
                     82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89,
                     90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57,
                     250,251,252,253,254,255)
             import string
             c.EBCDIC_TO_ASCII_MAP = string.maketrans( \
             ''.join(map(chr, range(256))), ''.join(map(chr, emap)))
         return s.translate(c.EBCDIC_TO_ASCII_MAP)
 

def BeautifulSoup.UnicodeDammit._subMSChar	(	self,
		orig
	)

private

Changes a MS smart quote character to an XML or HTML
entity.

Definition at line 1804 of file BeautifulSoup.py.

References BeautifulSoup.BeautifulStoneSoup.smartQuotesTo, and BeautifulSoup.UnicodeDammit.smartQuotesTo.

Referenced by BeautifulSoup.UnicodeDammit._convertFrom().

     def _subMSChar(self, orig):
         """Changes a MS smart quote character to an XML or HTML
         entity."""
         sub = self.MS_CHARS.get(orig)
         if isinstance(sub, tuple):
             if self.smartQuotesTo == 'xml':
                 sub = '&#x%s;' % sub[1]
             else:
                 sub = '&%s;' % sub[0]
         return sub
 

def BeautifulSoup.UnicodeDammit._toUnicode	(	self,
		data,
		encoding
	)

private

Given a string and its encoding, decodes the string into Unicode.
%encoding is a string recognized by encodings.aliases

Definition at line 1843 of file BeautifulSoup.py.

References BeautifulSoup.UnicodeDammit.unicode.

Referenced by BeautifulSoup.UnicodeDammit._convertFrom().

     def _toUnicode(self, data, encoding):
         '''Given a string and its encoding, decodes the string into Unicode.
         %encoding is a string recognized by encodings.aliases'''
 
         # strip Byte Order Mark (if present)
         if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
                and (data[2:4] != '\x00\x00'):
             encoding = 'utf-16be'
             data = data[2:]
         elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
                  and (data[2:4] != '\x00\x00'):
             encoding = 'utf-16le'
             data = data[2:]
         elif data[:3] == '\xef\xbb\xbf':
             encoding = 'utf-8'
             data = data[3:]
         elif data[:4] == '\x00\x00\xfe\xff':
             encoding = 'utf-32be'
             data = data[4:]
         elif data[:4] == '\xff\xfe\x00\x00':
             encoding = 'utf-32le'
             data = data[4:]
         newdata = unicode(data, encoding)
         return newdata
 

def BeautifulSoup.UnicodeDammit.find_codec	(	self,
		charset
	)

Definition at line 1936 of file BeautifulSoup.py.

References BeautifulSoup.UnicodeDammit._codec().

Referenced by BeautifulSoup.UnicodeDammit._convertFrom().

     def find_codec(self, charset):
         return self._codec(self.CHARSET_ALIASES.get(charset, charset)) \
                or (charset and self._codec(charset.replace("-", ""))) \
                or (charset and self._codec(charset.replace("-", "_"))) \
                or charset
 