various tree builder improvements. mostly ported from r1107, and a few others

--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%401212

Author: ryanking

Diff for: Manifest.txt

@@ -31,7 +31,7 @@ lib/html5/html5parser/in_table_body_phase.rb
 lib/html5/html5parser/in_table_phase.rb
 lib/html5/html5parser/initial_phase.rb
 lib/html5/html5parser/phase.rb
-lib/html5/html5parser/root_element_phase.rb
+lib/html5/html5parser/before_html_phase.rb
 lib/html5/html5parser/trailing_end_phase.rb
 lib/html5/inputstream.rb
 lib/html5/liberalxmlparser.rb

Diff for: lib/html5/constants.rb

@@ -12,6 +12,7 @@ def self._(str); str end
   ]
 
   SCOPING_ELEMENTS = %w[
+      applet
       button
       caption
       html
@@ -977,6 +978,8 @@ def self._(str); str end
       "unexpected-char-implies-table-voodoo" =>
          _("Unexpected non-space characters in " +
            "table context caused voodoo mode."),
+       "unpexted-hidden-input-in-table" =>
+        _("Unexpected input with type hidden in table context."),
       "unexpected-start-tag-implies-table-voodoo" =>
          _("Unexpected start tag (%(name)) in " +
            "table context caused voodoo mode."),
@@ -997,12 +1000,18 @@ def self._(str); str end
          _("Unexpected end tag (%(name)) in the table row phase. Ignored."),
       "unexpected-select-in-select" =>
          _("Unexpected select start tag in the select phase " +
-           "implies select start tag."),
+            "treated as select end tag."),
+      "unexpected-input-in-select" =>
+         _("Unexpected input start tag in the select phase."),
       "unexpected-start-tag-in-select" =>
          _("Unexpected start tag token (%(name)) in the select phase. " +
            "Ignored."),
       "unexpected-end-tag-in-select" =>
          _("Unexpected end tag (%(name)) in the select phase. Ignored."),
+      "unexpected-table-element-start-tag-in-select-in-table" =>
+         _("Unexpected table element start tag (%(name)s) in the select in table phase."),
+      "unexpected-table-element-end-tag-in-select-in-table" =>
+       _("Unexpected table element end tag (%(name)s) in the select in table phase."),
       "unexpected-char-after-body" =>
          _("Unexpected non-space characters in the after body phase."),
       "unexpected-start-tag-after-body" =>

Diff for: lib/html5/html5parser.rb

@@ -31,8 +31,8 @@ def self.parse_fragment(stream, options = {})
       new(options).parse_fragment(stream, container, encoding)
     end
 
-    @@phases = %w( initial rootElement beforeHead inHead afterHead inBody inTable inCaption
-      inColumnGroup inTableBody inRow inCell inSelect afterBody inFrameset afterFrameset trailingEnd )
+    @@phases = %w( initial beforeHtml beforeHead inHead afterHead inBody inTable inCaption
+      inColumnGroup inTableBody inRow inCell inSelect inSelectInTable afterBody inFrameset afterFrameset trailingEnd )
 
     # :strict - raise an exception when a parse error is encountered
     # :tree - a treebuilder class controlling the type of tree that will be
@@ -80,7 +80,7 @@ def _parse(stream, inner_html, encoding, container = 'div')
           @tokenizer.content_model_flag = :PCDATA
         end
 
-        @phase = @phases[:rootElement]
+        @phase = @phases[:beforeHtml]
         @phase.insert_html_element
         reset_insertion_mode
       else

Diff for: lib/html5/html5parser/before_head_phase.rb

@@ -12,6 +12,9 @@ def process_eof
       @parser.phase.process_eof
     end
 
+    def processSpaceCharacters(data)
+    end
+
     def processCharacters(data)
       startTagHead('head', {})
       @parser.phase.processCharacters(data)

Diff for: lib/html5/html5parser/root_element_phase.rb renamed to lib/html5/html5parser/before_html_phase.rb

@@ -1,7 +1,7 @@
 require 'html5/html5parser/phase'
 
 module HTML5
-  class RootElementPhase < Phase
+  class BeforeHtmlPhase < Phase
 
     def process_eof
       insert_html_element

Diff for: lib/html5/html5parser/in_body_phase.rb

@@ -6,12 +6,11 @@ class InBodyPhase < Phase
     # http://www.whatwg.org/specs/web-apps/current-work/#in-body
 
     handle_start 'html'
-    handle_start %w(base link meta script style) => 'ProcessInHead'
-    handle_start 'title'
+    handle_start %w(base link meta script style title) => 'ProcessInHead'
 
     handle_start 'body', 'form', 'plaintext', 'a', 'button', 'xmp', 'table', 'hr', 'image'
 
-    handle_start 'input', 'textarea', 'select', 'isindex', %w(marquee object)
+    handle_start 'input', 'textarea', 'select', 'isindex', %w(applet marquee object)
 
     handle_start %w(li dd dt) => 'ListItem'
 
@@ -28,7 +27,7 @@ class InBodyPhase < Phase
 
     handle_start %w(event-source section nav article aside header footer datagrid command) => 'New'
 
-    handle_end 'p', 'body', 'html', 'form', %w(button marquee object), %w(dd dt li) => 'ListItem'
+    handle_end 'p', 'body', 'html', 'form', %w(applet button marquee object), %w(dd dt li) => 'ListItem'
 
     handle_end %w(address blockquote center div dl fieldset listing menu ol pre ul) => 'Block'
 
@@ -64,7 +63,7 @@ class << self
       end
 
       if (data.length > 0 and data[0] == ?\n && 
-        %w[pre textarea].include?(@tree.open_elements.last.name) && !@tree.open_elements.last.hasContent)
+        %w[listing pre textarea].include?(@tree.open_elements.last.name) && !@tree.open_elements.last.hasContent)
         data = data[1..-1]
       end
 
@@ -91,11 +90,6 @@ def startTagProcessInHead(name, attributes)
       @parser.phases[:inHead].processStartTag(name, attributes)
     end
 
-    def startTagTitle(name, attributes)
-      parse_error("unexpected-start-tag-out-of-my-head", {"name" => name})
-      @parser.phases[:inHead].processStartTag(name, attributes)
-    end
-
     def startTagBody(name, attributes)
       parse_error("unexpected-start-tag", {"name" => "body"})
 
@@ -113,7 +107,7 @@ def startTagBody(name, attributes)
     def startTagCloseP(name, attributes)
       endTagP('p') if in_scope?('p')
       @tree.insert_element(name, attributes)
-      if name == 'pre'
+      if ['pre', 'listing'].include?(name) 
         class << self
           remove_method :processSpaceCharacters rescue nil
           alias processSpaceCharacters processSpaceCharactersDropNewline
@@ -220,7 +214,7 @@ def startTagButton(name, attributes)
       end
     end
 
-    def startTagMarqueeObject(name, attributes)
+    def startTagAppletMarqueeObject(name, attributes)
       @tree.reconstructActiveFormattingElements
       @tree.insert_element(name, attributes)
       @tree.activeFormattingElements.push(Marker)
@@ -303,7 +297,14 @@ def startTagCdata(name, attributes)
     def startTagSelect(name, attributes)
       @tree.reconstructActiveFormattingElements
       @tree.insert_element(name, attributes)
-      @parser.phase = @parser.phases[:inSelect]
+      
+      if [@parser.phases[:inTable], @parser.phases[:inCaption],
+        @parser.phases[:inColumnGroup], @parser.phases[:inTableBody], @parser.phases[:inRow],
+        @parser.phases[:inCell]].include?(@parser.phase)
+        @parser.phase = @parser.phases[:inSelectInTable]
+      else
+        @parser.phase = @parser.phases[:inSelect]
+      end
     end
 
     def startTagMisplaced(name, attributes)
@@ -528,7 +529,7 @@ def endTagFormatting(name)
       end
     end
 
-    def endTagButtonMarqueeObject(name)
+    def endTagAppletButtonMarqueeObject(name)
       @tree.generateImpliedEndTags if in_scope?(name)
 
       unless @tree.open_elements.last.name == name

Diff for: lib/html5/html5parser/in_head_phase.rb

@@ -33,20 +33,24 @@ def startTagHead(name, attributes)
     end
 
     def startTagTitle(name, attributes)
-      element = @tree.createElement(name, attributes)
-      appendToHead(element)
-      @tree.open_elements.push(element)
+      if @tree.head_pointer != nil# && @parser.phase == @parser.phases[:inHead]
+        element = @tree.createElement(name, attributes)
+        appendToHead(element)
+        @tree.open_elements << element
+      else
+        @tree.insert_element(name, attributes)
+      end
       @parser.tokenizer.content_model_flag = :RCDATA
     end
 
     def startTagStyle(name, attributes)
-      element = @tree.createElement(name, attributes)
       if @tree.head_pointer != nil and @parser.phase == @parser.phases[:inHead]
+        element = @tree.createElement(name, attributes)
         appendToHead(element)
+        @tree.open_elements.push(element)
       else
-        @tree.open_elements.last.appendChild(element)
+        @tree.insert_element(name, attributes)
       end
-      @tree.open_elements.push(element)
       @parser.tokenizer.content_model_flag = :CDATA
     end
 
@@ -65,7 +69,7 @@ def startTagNoscript(name, attributes)
     def startTagScript(name, attributes)
       #XXX Inner HTML case may be wrong
       element = @tree.createElement(name, attributes)
-      element._flags.push("parser-inserted")
+      element.flags.push("parser-inserted")
       if @tree.head_pointer != nil and @parser.phase == @parser.phases[:inHead]
         appendToHead(element)
       else
@@ -76,11 +80,12 @@ def startTagScript(name, attributes)
     end
 
     def startTagBaseLinkMeta(name, attributes)
-      element = @tree.createElement(name, attributes)
       if @tree.head_pointer != nil and @parser.phase == @parser.phases[:inHead]
+        element = @tree.createElement(name, attributes)
         appendToHead(element)
       else
-        @tree.open_elements.last.appendChild(element)
+        @tree.insert_element(name, attributes)
+        @tree.open_elements.pop
       end
     end
 

Diff for: lib/html5/html5parser/in_row_phase.rb

@@ -13,6 +13,10 @@ def processCharacters(data)
       @parser.phases[:inTable].processCharacters(data)
     end
 
+    def processSpaceCharacters(data)
+      @parser.phases[:inTable].processSpaceCharacters(data)
+    end
+
     def startTagTableCell(name, attributes)
       clearStackToTableRowContext
       @tree.insert_element(name, attributes)

Diff for: lib/html5/html5parser/in_select_phase.rb

@@ -6,7 +6,7 @@ class InSelectPhase < Phase
     # http://www.whatwg.org/specs/web-apps/current-work/#in-select
 
     handle_start 'html', 'option', 'optgroup', 'select'
-
+    handle_start 'input'
     handle_end 'option', 'optgroup', 'select', %w( caption table tbody tfoot thead tr td th ) => 'TableElements'
 
     def processCharacters(data)
@@ -30,6 +30,11 @@ def startTagSelect(name, attributes)
       endTagSelect('select')
     end
 
+    def startTagInput(name, attributes)
+      @parser.parse_error("unexpected-input-in-select")
+      endTagSelect("select")
+      @parser.phase.processStartTag(name, attributes)
+    end
     def startTagOther(name, attributes)
       parse_error("unexpected-start-tag-in-select", {"name" => name})
     end

Diff for: lib/html5/html5parser/in_table_body_phase.rb

@@ -13,6 +13,10 @@ def processCharacters(data)
       @parser.phases[:inTable].processCharacters(data)
     end
 
+    def processSpaceCharacters(data)
+      @parser.phases[:inTable].processSpaceCharacters(data)
+    end
+
     def startTagTr(name, attributes)
       clearStackToTableBodyContext
       @tree.insert_element(name, attributes)