""" While there unquestionably are a lot of applications of XML for which
strict, draconian, error handing is appropriate, there also are a number
of use cases for which robust scavenging is required, as is evidenced by
the popularity of libraries such as BeautifulSoup and the Universal Feed
Parser.

HTML5's grammar is a rich a blend of SGML (the common ancestor to both HTML
and XML), XML, and custom parsing rules; these rules were arrived at by
observing the effective consensus by browser vendors have converged on in
the process of dealing with the enormous diversity of documents that exist
on the internet; often produced by hand editing and copy/pasting templates.

Much of that experience can directly benefit those that find themselves in
need of recovering data from mal-formed XML at any cost, particularly for
the XML documents which are produced using similar hand editing, copy/pasting,
and templating techniques that are used to produce invalid HTML.
Additionally, given the rough similarity between HTML and XML syntax, naive
users will often copy things that happen to work in HTML into XML documents.

Just be aware that in scavenge mode, some data will be interpreted in manners
different than the author intended, as such intent can't be determined.  Also
be aware that some of the more advanced XML features that are less commonly
used in hand-produced XML, like internal DTD subsets, are not supported by
this process.  For this reason, it is recommended that data first be parsed
by a "real" XML parser and this logic only be used as a fallback.

References:
 * http://googlereader.blogspot.com/2005/12/xml-errors-in-feeds.html
 * http://wiki.whatwg.org/wiki/HtmlVsXhtml

@@TODO:
 * Build a Treebuilder that produces Python DOM objects:
     http://docs.python.org/lib/module-xml.dom.html
 * Produce SAX events based on the produced DOM.  This is intended not to
   support streaming, but rather to support application level compatibility. 
 * Optional namespace support
 * Special case the output of XHTML <script> elements so that the empty
   element syntax is never used, even when the src attribute is provided.
   Also investigate the use of <![CDATA[]>> to ensure dual HTML/XHTML
   compatibility.
 * Map illegal XML characters to U+FFFD, possibly with additional markup in
   the case of XHTML
 * Selectively lowercase only XHTML, but not foreign markup
"""

from xml.sax.saxutils import escape
import html5parser, treebuilders
import gettext
_ = gettext.gettext

class XHTMLParser(html5parser.HTMLParser):
    """ liberal XMTHML parser """

    def parse(self, *args, **kwargs):
        document = html5parser.HTMLParser.parse(self, *args, **kwargs)

        # default to the XHTML namespace
        for node in document.childNodes:
            if isinstance(node, XmlElement) and node.name == 'html':
                if not node.attributes.has_key('xmlns'):
                    node.attributes['xmlns'] = 'http://www.w3.org/1999/xhtml'
                    break

        return document

    def normalizeToken(self, token):
        if token["type"] == "StartTag" or token["type"] == "EmptyTag":
            # We need to remove the duplicate attributes and convert attributes
            # to a dict so that [["x", "y"], ["x", "z"]] becomes {"x": "y"}

            # AT When Python 2.4 is widespread we should use
            # dict(reversed(token.data))
            token["data"] = dict(token["data"][::-1])

            # For EmptyTags, process both a Start and an End tag
            if token["type"] == "EmptyTag":
                self.phase.processStartTag(token["name"], token["data"])
                token["data"] = {}
                token["type"] = "EndTag"

        return token

class XMLParser(XHTMLParser):
    """ liberal XML parser """

    def __init__(self, *args, **kwargs):
        XHTMLParser.__init__(self, *args, **kwargs)
        self.phases["initial"] = XmlRootPhase(self, self.tree)

class XmlRootPhase(html5parser.Phase):
    """ Prime the Xml parser """
    def __getattr__(self, name):
        self.tree.openElements.append(self.tree.document)
        self.parser.phase = XmlElementPhase(self.parser, self.tree)
        return getattr(self.parser.phase, name)

class XmlElementPhase(html5parser.Phase):
    """ Generic handling for all XML elements """

    def __init__(self, *args, **kwargs):
        html5parser.Phase.__init__(self, *args, **kwargs)
        self.startTagHandler = html5parser.utils.MethodDispatcher([])
        self.startTagHandler.default = self.startTagOther
        self.endTagHandler = html5parser.utils.MethodDispatcher([])
        self.endTagHandler.default = self.endTagOther

    def startTagOther(self, name, attributes):
        element = self.tree.createElement(name, attributes)
        self.tree.openElements[-1].appendChild(element)
        self.tree.openElements.append(element)

    def endTagOther(self, name):
        for node in self.tree.openElements[::-1]:
            if node.name == name:
                self.tree.generateImpliedEndTags()
                if self.tree.openElements[-1].name != name:
                    self.parser.parseError(_("Unexpected end tag " + name +\
                      "."))
                while self.tree.openElements.pop() != node:
                    pass
                break
            else:
                self.parser.parseError()

    def processCharacters(self, data):
        self.tree.insertText(data)

class XmlDocument(treebuilders.simpletree.Document):
    """ an XML document that contains a version of xml.dom.minidom's toxml()
        method """
    def toxml(self, encoding="utf=8"):
        result = ''
        for child in self.childNodes:
            if isinstance(child, treebuilders.simpletree.TextNode):
                result += escape(child.value)
            else:
                result += child.toxml()
        return result.encode(encoding)

class XmlElement(treebuilders.simpletree.Element):
    """ implement toxml() for XML Elements """
    def toxml(self):
        result = '<' + self.name
        if self.attributes:
            for name,value in self.attributes.iteritems():
                result += ' %s="%s"' % (name, escape(value,{'"':'&quot;'}))
        if self.childNodes:
            result += '>'
            for child in self.childNodes:
                if isinstance(child, treebuilders.simpletree.TextNode):
                    result += escape(child.value)
                else:
                    result += child.toxml()
            result += '</%s>' % self.name
        else:
            result += '/>'
        return result

class XmlComment(treebuilders.simpletree.CommentNode):
    """ implement toxml() at for XML comments """
    def toxml(self):
        return '<!--%s-->' % self.data

class XTreeBuilder(treebuilders.simpletree.TreeBuilder):
    """ build an XML tree """
    documentClass = XmlDocument
    elementClass = XmlElement
    commentClass = XmlComment