import unittest, new, re

class HtmlHeuristicsTest(unittest.TestCase): pass

testcases = 0
def test(str, cond):
  global testcases
  testcases += 1
  func = lambda self: getattr(self,cond)(lookslikehtml(str), cond+": "+str)
  method = new.instancemethod(func, None, HtmlHeuristicsTest)
  setattr(HtmlHeuristicsTest, "test_%d" % testcases, method)

# each call to HTML adds a test case to HtmlHeuristicsTest
def HTML(a): test(a, 'assertTrue')
def NONHTML(a): test(a, 'assertFalse')

acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'b', 'big',
    'blockquote', 'br', 'button', 'caption', 'center', 'cite', 'code', 'col',
    'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', 'em', 'fieldset',
    'font', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img',
    'input', 'ins', 'kbd', 'label', 'legend', 'li', 'map', 'menu', 'ol',
    'optgroup', 'option', 'p', 'pre', 'q', 's', 'samp', 'select', 'small',
    'span', 'strike', 'strong', 'sub', 'sup', 'table', 'tbody', 'td',
    'textarea', 'tfoot', 'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var']

# a number of elements in a number of RSS variants are nominally plain
# text, but this is routinely ignored.  This is an attempt to detect
# the most common cases.  As false positives often result in silent
# data loss, this function errs on the conservative side. 
def lookslikehtml(str):
  # must have a close tag or a entity reference to qualify
  if not (re.search(r'</(\w+)>',str) or re.search("&#?\w+;",str)): return False

  # all tags must be in a restricted subset of valid HTML tags
  if filter(lambda t: t.lower() not in acceptable_elements, 
    re.findall(r'</?(\w+)',str)): return False

  return True

# Remember: these tests represent strings AFTER a first level of XML unescaping

# definately not
NONHTML('plain text')
NONHTML('2 < 3')
NONHTML('AT&T')

# not quite
NONHTML('<code>')
NONHTML(' <rss> ... </rss> ')
NONHTML('http://example.com?q=foo&order=desc')

# looks like HTML
HTML('<a href="">here</a>')
HTML('<B>BOLD</B>')
HTML('<i>italics</i>')
HTML('&copy; 2006')
HTML('&#xA9; 2006')
HTML('&#169; 2006')

if __name__ == '__main__':
    unittest.main()