Fix a bug that Stream parser doesn't expand the user-defined entity references for "text"

lib/rexml/parsers/streamparser.rb

@@ -7,12 +7,17 @@ class StreamParser
       def initialize source, listener
         @listener = listener
         @parser = BaseParser.new( source )
+        @entities = {}
       end
 
       def add_listener( listener )
         @parser.add_listener( listener )
       end
 
+      def entity_expansion_count
+        @parser.entity_expansion_count
+      end
+
       def parse
         # entity string
         while true
@@ -28,7 +33,7 @@ def parse
           when :end_element
             @listener.tag_end( event[1] )
           when :text
-            unnormalized = @parser.unnormalize( event[1] )
+            unnormalized = @parser.unnormalize( event[1], @entities )
             @listener.text( unnormalized )
           when :processing_instruction
             @listener.instruction( *event[1,2] )
@@ -40,6 +45,7 @@ def parse
           when :comment, :attlistdecl, :cdata, :xmldecl, :elementdecl
             @listener.send( event[0].to_s, *event[1..-1] )
           when :entitydecl, :notationdecl
+            @entities[ event[1] ] = event[2] if event.size == 3
             @listener.send( event[0].to_s, event[1..-1] )
           when :externalentity
             entity_reference = event[1]

test/test_stream.rb

@@ -87,6 +87,40 @@ def entity(content)
 
       assert_equal(["ISOLat2"], listener.entities)
     end
+
+    def test_entity_replacement
+      source = '<!DOCTYPE foo [
+      <!ENTITY la "1234">
+      <!ENTITY lala "--&la;--">
+      <!ENTITY lalal "&la;&la;">
+      ]><a><la>&la;</la><lala>&lala;</lala></a>'
+
+      listener = MyListener.new
+      class << listener
+        attr_accessor :text_values
+        def text(text)
+          @text_values << text
+        end
+      end
+      listener.text_values = []
+      REXML::Document.parse_stream(source, listener)
+      assert_equal(["1234", "--1234--"], listener.text_values)
+    end
+
+    def test_characters_predefined_entities
+      source = '<root><a>&lt;P&gt; &lt;I&gt; &lt;B&gt; Text &lt;/B&gt;  &lt;/I&gt;</a></root>'
+
+      listener = MyListener.new
+      class << listener
+        attr_accessor :text_value
+        def text(text)
+          @text_value << text
+        end
+      end
+      listener.text_value = ""
+      REXML::Document.parse_stream(source, listener)
+      assert_equal("<P> <I> <B> Text </B>  </I>", listener.text_value)
+    end
   end
 
   class EntityExpansionLimitTest < Test::Unit::TestCase
@@ -100,6 +134,81 @@ def teardown
       REXML::Security.entity_expansion_text_limit = @default_entity_expansion_text_limit
     end
 
+    def test_have_value
+      source = <<-XML
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE member [
+  <!ENTITY a "&b;&b;&b;&b;&b;&b;&b;&b;&b;&b;">
+  <!ENTITY b "&c;&c;&c;&c;&c;&c;&c;&c;&c;&c;">
+  <!ENTITY c "&d;&d;&d;&d;&d;&d;&d;&d;&d;&d;">
+  <!ENTITY d "&e;&e;&e;&e;&e;&e;&e;&e;&e;&e;">
+  <!ENTITY e "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx">
+]>
+<member>
+&a;
+</member>
+      XML
+
+      assert_raise(RuntimeError.new("entity expansion has grown too large")) do
+        REXML::Document.parse_stream(source, MyListener.new)
+      end
+    end
+
+    def test_empty_value
+      source = <<-XML
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE member [
+  <!ENTITY a "&b;&b;&b;&b;&b;&b;&b;&b;&b;&b;">
+  <!ENTITY b "&c;&c;&c;&c;&c;&c;&c;&c;&c;&c;">
+  <!ENTITY c "&d;&d;&d;&d;&d;&d;&d;&d;&d;&d;">
+  <!ENTITY d "&e;&e;&e;&e;&e;&e;&e;&e;&e;&e;">
+  <!ENTITY e "">
+]>
+<member>
+&a;
+</member>
+      XML
+
+      listener = MyListener.new
+      REXML::Security.entity_expansion_limit = 100000
+      parser = REXML::Parsers::StreamParser.new( source, listener )
+      parser.parse
+      assert_equal(11111, parser.entity_expansion_count)
+
+      REXML::Security.entity_expansion_limit = @default_entity_expansion_limit
+      parser = REXML::Parsers::StreamParser.new( source, listener )
+      assert_raise(RuntimeError.new("number of entity expansions exceeded, processing aborted.")) do
+        parser.parse
+      end
+      assert do
+        parser.entity_expansion_count > @default_entity_expansion_limit
+      end
+    end
+
+    def test_with_default_entity
+      source = <<-XML
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE member [
+  <!ENTITY a "a">
+  <!ENTITY a2 "&a; &a;">
+]>
+<member>
+&a;
+&a2;
+&lt;
+</member>
+      XML
+
+      listener = MyListener.new
+      REXML::Security.entity_expansion_limit = 4
+      REXML::Document.parse_stream(source, listener)
+
+      REXML::Security.entity_expansion_limit = 3
+      assert_raise(RuntimeError.new("number of entity expansions exceeded, processing aborted.")) do
+        REXML::Document.parse_stream(source, listener)
+      end
+    end
+
     def test_with_only_default_entities
       member_value = "&lt;p&gt;#{'A' * @default_entity_expansion_text_limit}&lt;/p&gt;"
       source = <<-XML
@@ -117,14 +226,42 @@ def text(text)
         end
       end
       listener.text_value = ""
-      REXML::Document.parse_stream(source, listener)
+      parser = REXML::Parsers::StreamParser.new( source, listener )
+      parser.parse
 
       expected_value = "<p>#{'A' * @default_entity_expansion_text_limit}</p>"
       assert_equal(expected_value, listener.text_value.strip)
+      assert_equal(0, parser.entity_expansion_count)
       assert do
         listener.text_value.bytesize > @default_entity_expansion_text_limit
       end
     end
+
+    def test_entity_expansion_text_limit
+      source = <<-XML
+<!DOCTYPE member [
+  <!ENTITY a "&b;&b;&b;">
+  <!ENTITY b "&c;&d;&e;">
+  <!ENTITY c "xxxxxxxxxx">
+  <!ENTITY d "yyyyyyyyyy">
+  <!ENTITY e "zzzzzzzzzz">
+]>
+<member>&a;</member>
+      XML
+
+      listener = MyListener.new
+      class << listener
+        attr_accessor :text_value
+        def text(text)
+          @text_value << text
+        end
+      end
+      listener.text_value = ""
+      REXML::Security.entity_expansion_text_limit = 90
+      REXML::Document.parse_stream(source, listener)
+
+      assert_equal(90, listener.text_value.size)
+    end
   end
 
   # For test_listener