puts File.read('tokenizer.pre')

python = File.read('/home/rubys/git/xml5/lib.py/tokenizer.py')

python.gsub!(/i\s*=\s*0\s+
  for\s+x\s+in\s+entity\[1\]:\s+
  self.stream.queue.insert\(i,\s*x\)\s+
  i\s*\+=\s*1/x, 'buffer.unget(entity[1])')

continue = nil
python.split(/^    (#.*\n|def )/).each do |method|
  puts method.sub(/^#/, '//') if method =~ /^#/
  dcl, body =  method.split("\n",2)
  next unless dcl =~ /State\(self\):$/
  name = dcl.split('(').first.gsub(/[A-Z]/) {|c| "_#{c.downcase}"}

  indent = 4
  vars = []

  puts "t.prototype.#{name} = function(buffer) {"
  body.split("\n").each do |line|
    next if line.empty?

    spaces = line.slice!(/^ +/)
    spaces.sub! /^    /, ''
    while indent > spaces.length
        indent -= 4
        break if indent == spaces.length and line =~ /^el(se:|if )/
        puts (" "*indent).gsub('    ',"\t") + '}'
    end
    if line.sub! /^#/, '//'
      puts spaces.gsub('    ',"\t") + line.strip
      next
    end

    if line =~ /^(\w+)\s*=/ and not vars.include?($1)
      vars << $1
      line = "var #{line}"
    end

    line.sub! /^else:/, '} else {'
    line.sub! /^elif /, '} else if('
    line.sub! /^if /, 'if('
    line.sub! /^for /, 'for('
    line.sub! /:$/, ') {'
    line.sub! /\bcurrentToken\b/, 'current_token'
    line.sub! /\bconsumeEntity\(.*?\)/, 'consume_entity(buffer)'
    line.gsub! /\bdata\b/, 'c'
    line.sub! /"c"/, '"data"'
    line.sub! /current_token\["data"\]/, 'current_token.data'
    line.gsub! /\bself\./, 'this.'
    line.gsub! /\bthis\.stream\b/, 'buffer'
    line.sub! /buffer\.queue\.append/, 'buffer.unget'
    line.sub! /\bor\b/, '||'
    line.sub! /False/, 'false'
    line.sub! /True/, 'true'
    line.gsub! /,"(\w+)": ?/, ', \1: '
    line.gsub! /"(\w+)": ?/, '\1: '
    line.sub! 'this.tokenQueue.append', 'this.emitToken'
    line.sub! /not EOF in charStack/, 'charStack[charStack.length-1] != EOF'
    line.sub! /\bEOF\b/, 'XML5.EOF'
    line.sub! /c in spaceCharacters/, 'XML5.SPACE_CHARACTERS_R.test(c)'
    line.sub! 'entity[0] == "Characters"', 'entity'
    line.sub! 'entity[1]', 'entity'
    line.sub! /raise (\w+)Error/, 'throw(new Error("\1"))'
    line.sub! '[:-1]', '.slice(0,-1)'
    line.sub! /(\w) in xrange\((\d+)\)/, 'var \1=0; \1<\2; \1++'
    line.sub! 'emitCurrentToken', 'emit_current_token'
    line.sub! '.append(', '.push('
    line.sub! '.extend(', '.concat('
    line.sub! /"(.*?)"\.join\((.*?)\)/, '\2.join("\1")'
    line.gsub! '[-1]', '.last()'

    line.sub! /frozenset\(\(.*?\)\)/ do |c|
      line.scan(/".*?"/).map {|c| c[1..-2]}.join.inspect
    end

    line.sub! /\s*==\s*\[".*?"\]/ do |c|
      '.join("") == ' + line.scan(/".*?"/).map {|c| c[1..-2]}.join.inspect
    end

    line.sub! /charsUntil\(\(.*?\)\)/ do |c|
      "charsUntil(#{line.scan(/".*?"/).map {|c| c[1..-2]}.join.inspect})"
    end

    line.sub! /charsUntil\(.*?\)/ do |c|
      c.sub! 'chars', 'match'
      c.sub! /spaceCharacters/, 'XML5.SPACE_CHARACTERS_IN'
      if c.include?('|')
        c.sub(/\(.*?\)/) do |expr|
          '("[" + ' + expr[1..-2].gsub('|','+') + ' + "]")'
        end
      elsif c.include?(', true')
        c.sub(', true', ' /*, true */')
      elsif c =~ /\("..+"\)/
        c.sub(/\(.*?\)/) { |expr| '("[' + expr[2..-3] + ']")'}
      else
        c
      end
    end

    line.sub! /this\.states\[(.*?)\]/ do |c|
      $1.gsub(/[A-Z]/) {|c| "_#{c.downcase}"}.gsub(/(['"])$/, '_state\1')
    end
    line += ';' unless line =~ /[{]$/
    indent = continue || spaces.length
    continue = (line =~ /\\;?$/ ? indent : nil)
    line.sub! /\\;$/, ''
    puts spaces.gsub('    ',"\t") + line.strip
  end
  puts '}'
  puts
end

print File.read('tokenizer.post')