Port liquid-c bug compatible whitespace trimming

The parse_context no longer needs to be passed in because it is available
through through an attr_reader on the instance. However, the markup and
tokenizer weren't made available.  This refactor also makes the parameters
given to Document#unknown_tag consistent with Block#unknown_tag.

lib/liquid/block.rb

@@ -4,8 +4,13 @@ module Liquid
   class Block < Tag
     MAX_DEPTH = 100
 
+    def initialize(tag_name, markup, options)
+      super
+      @blank = true
+    end
+
     def parse(tokens)
-      @body = BlockBody.new
+      @body = new_body
       while parse_body(@body, tokens)
       end
     end
@@ -15,12 +20,16 @@ module Liquid
       @body.render(context)
     end
 
+    def blank?
+      @blank
+    end
+
     def nodelist
       @body.nodelist
     end
 
-    def unknown_tag(tag, _params, _tokens)
-      Block.raise_unknown_tag(tag, block_name, block_delimiter, parse_context)
+    def unknown_tag(tag_name, _markup, _tokenizer)
+      Block.raise_unknown_tag(tag_name, block_name, block_delimiter, parse_context)
     end
 
     # @api private
@@ -46,8 +55,14 @@ module Liquid
       @block_delimiter ||= "end#{block_name}"
     end
 
-    protected
+    private
 
+    # @api public
+    def new_body
+      BlockBody.new
+    end
+
+    # @api public
     def parse_body(body, tokens)
       if parse_context.depth >= MAX_DEPTH
         raise StackLevelError, "Nesting too deep"
@@ -55,6 +70,8 @@ module Liquid
       parse_context.depth += 1
       begin
         body.parse(tokens, parse_context) do |end_tag_name, end_tag_params|
+          @blank &&= body.blank?
+
           return false if end_tag_name == block_delimiter
           unless end_tag_name
             raise SyntaxError, parse_context.locale.t("errors.syntax.tag_never_closed", block_name: block_name)

lib/liquid/block_body.rb

@@ -58,6 +58,28 @@ module Liquid
       Block.raise_unknown_tag(tag, 'liquid', '%}', parse_context)
     end
 
+    # @api private
+    def self.raise_missing_tag_terminator(token, parse_context)
+      raise SyntaxError, parse_context.locale.t("errors.syntax.tag_termination", token: token, tag_end: TagEnd.inspect)
+    end
+
+    # @api private
+    def self.raise_missing_variable_terminator(token, parse_context)
+      raise SyntaxError, parse_context.locale.t("errors.syntax.variable_termination", token: token, tag_end: VariableEnd.inspect)
+    end
+
+    # @api private
+    def self.render_node(context, output, node)
+      node.render_to_output_buffer(context, output)
+    rescue UndefinedVariable, UndefinedDropMethod, UndefinedFilter => e
+      context.handle_error(e, node.line_number)
+    rescue MemoryError
+      raise
+    rescue ::StandardError => e
+      line_number = node.is_a?(String) ? nil : node.line_number
+      output << context.handle_error(e, line_number)
+    end
+
     private def parse_liquid_tag(markup, parse_context)
       liquid_tag_tokenizer = Tokenizer.new(markup, line_number: parse_context.line_number, for_liquid_tag: true)
       parse_for_liquid_tag(liquid_tag_tokenizer, parse_context) do |end_tag_name, _end_tag_markup|
@@ -74,7 +96,7 @@ module Liquid
         when token.start_with?(TAGSTART)
           whitespace_handler(token, parse_context)
           unless token =~ FullToken
-            raise_missing_tag_terminator(token, parse_context)
+            BlockBody.raise_missing_tag_terminator(token, parse_context)
           end
           tag_name = Regexp.last_match(2)
           markup   = Regexp.last_match(4)
@@ -120,7 +142,11 @@ module Liquid
       if token[2] == WhitespaceControl
         previous_token = @nodelist.last
         if previous_token.is_a?(String)
+          first_byte = previous_token.getbyte(0)
           previous_token.rstrip!
+          if previous_token.empty? && parse_context[:bug_compatible_whitespace_trimming] && first_byte
+            previous_token << first_byte
+          end
         end
       end
       parse_context.trim_whitespace = (token[-3] == WhitespaceControl)
@@ -155,12 +181,10 @@ module Liquid
     end
 
     def render_to_output_buffer(context, output)
-      context.resource_limits.render_score += @nodelist.length
+      context.resource_limits.increment_render_score(@nodelist.length)
 
       idx = 0
       while (node = @nodelist[idx])
-        previous_output_size = output.bytesize
-
         if node.instance_of?(String)
           output << node
         else
@@ -172,7 +196,7 @@ module Liquid
         end
         idx += 1
 
-        raise_if_resource_limits_reached(context, output.bytesize - previous_output_size)
+        context.resource_limits.increment_write_score(output)
       end
 
       output
@@ -181,18 +205,7 @@ module Liquid
     private
 
     def render_node(context, output, node)
-      node.render_to_output_buffer(context, output)
-    rescue UndefinedVariable, UndefinedDropMethod, UndefinedFilter => e
-      context.handle_error(e, node.line_number)
-    rescue ::StandardError => e
-      line_number = node.is_a?(String) ? nil : node.line_number
-      output << context.handle_error(e, line_number)
-    end
-
-    def raise_if_resource_limits_reached(context, length)
-      context.resource_limits.render_length += length
-      return unless context.resource_limits.reached?
-      raise MemoryError, "Memory limits exceeded"
+      BlockBody.render_node(context, output, node)
     end
 
     def create_variable(token, parse_context)
@@ -200,15 +213,17 @@ module Liquid
         markup = content.first
         return Variable.new(markup, parse_context)
       end
-      raise_missing_variable_terminator(token, parse_context)
+      BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
 
+    # @deprecated Use {.raise_missing_tag_terminator} instead
     def raise_missing_tag_terminator(token, parse_context)
-      raise SyntaxError, parse_context.locale.t("errors.syntax.tag_termination", token: token, tag_end: TagEnd.inspect)
+      BlockBody.raise_missing_tag_terminator(token, parse_context)
     end
 
+    # @deprecated Use {.raise_missing_variable_terminator} instead
     def raise_missing_variable_terminator(token, parse_context)
-      raise SyntaxError, parse_context.locale.t("errors.syntax.variable_termination", token: token, tag_end: VariableEnd.inspect)
+      BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
 
     def registered_tags

lib/liquid/document.rb

@@ -1,23 +1,33 @@
 # frozen_string_literal: true
 
 module Liquid
-  class Document < BlockBody
+  class Document
     def self.parse(tokens, parse_context)
-      doc = new
+      doc = new(parse_context)
       doc.parse(tokens, parse_context)
       doc
     end
 
-    def parse(tokens, parse_context)
-      super do |end_tag_name, _end_tag_params|
-        unknown_tag(end_tag_name, parse_context) if end_tag_name
+    attr_reader :parse_context, :body
+
+    def initialize(parse_context)
+      @parse_context = parse_context
+      @body = new_body
+    end
+
+    def nodelist
+      @body.nodelist
+    end
+
+    def parse(tokenizer, parse_context)
+      while parse_body(tokenizer)
       end
     rescue SyntaxError => e
       e.line_number ||= parse_context.line_number
       raise
     end
 
-    def unknown_tag(tag, parse_context)
+    def unknown_tag(tag, _markup, _tokenizer)
       case tag
       when 'else', 'end'
         raise SyntaxError, parse_context.locale.t("errors.syntax.unexpected_outer_tag", tag: tag)
@@ -25,5 +35,30 @@ module Liquid
         raise SyntaxError, parse_context.locale.t("errors.syntax.unknown_tag", tag: tag)
       end
     end
+
+    def render_to_output_buffer(context, output)
+      @body.render_to_output_buffer(context, output)
+    end
+
+    def render(context)
+      @body.render(context)
+    end
+
+    private
+
+    def new_body
+      Liquid::BlockBody.new
+    end
+
+    def parse_body(tokenizer)
+      @body.parse(tokenizer, parse_context) do |unknown_tag_name, unknown_tag_markup|
+        if unknown_tag_name
+          unknown_tag(unknown_tag_name, unknown_tag_markup, tokenizer)
+          true
+        else
+          false
+        end
+      end
+    end
   end
 end

lib/liquid/resource_limits.rb

@@ -2,8 +2,8 @@
 
 module Liquid
   class ResourceLimits
-    attr_accessor :render_length, :render_score, :assign_score,
-      :render_length_limit, :render_score_limit, :assign_score_limit
+    attr_accessor :render_length_limit, :render_score_limit, :assign_score_limit
+    attr_reader :render_score, :assign_score
 
     def initialize(limits)
       @render_length_limit = limits[:render_length_limit]
@@ -12,14 +12,51 @@ module Liquid
       reset
     end
 
+    def increment_render_score(amount)
+      @render_score += amount
+      raise_limits_reached if @render_score_limit && @render_score > @render_score_limit
+    end
+
+    def increment_assign_score(amount)
+      @assign_score += amount
+      raise_limits_reached if @assign_score_limit && @assign_score > @assign_score_limit
+    end
+
+    # update either render_length or assign_score based on whether or not the writes are captured
+    def increment_write_score(output)
+      if (last_captured = @last_capture_length)
+        captured = output.bytesize
+        increment = captured - last_captured
+        @last_capture_length = captured
+        increment_assign_score(increment)
+      elsif @render_length_limit && output.bytesize > @render_length_limit
+        raise_limits_reached
+      end
+    end
+
+    def raise_limits_reached
+      @reached_limit = true
+      raise MemoryError, "Memory limits exceeded"
+    end
+
     def reached?
-      (@render_length_limit && @render_length > @render_length_limit) ||
-        (@render_score_limit && @render_score > @render_score_limit) ||
-        (@assign_score_limit && @assign_score > @assign_score_limit)
+      @reached_limit
     end
 
     def reset
-      @render_length = @render_score = @assign_score = 0
+      @reached_limit = false
+      @last_capture_length = nil
+      @render_score = @assign_score = 0
+    end
+
+    def with_capture
+      old_capture_length = @last_capture_length
+      begin
+        @last_capture_length = 0
+        yield
+      ensure
+        @last_capture_length = old_capture_length
+      end
     end
   end
 end

lib/liquid/tags/assign.rb

@@ -27,7 +27,7 @@ module Liquid
     def render_to_output_buffer(context, output)
       val = @from.render(context)
       context.scopes.last[@to] = val
-      context.resource_limits.assign_score += assign_score_of(val)
+      context.resource_limits.increment_assign_score(assign_score_of(val))
       output
     end
 

lib/liquid/tags/capture.rb

@@ -25,9 +25,10 @@ module Liquid
     end
 
     def render_to_output_buffer(context, output)
-      capture_output = render(context)
-      context.scopes.last[@to] = capture_output
-      context.resource_limits.assign_score += capture_output.bytesize
+      context.resource_limits.with_capture do
+        capture_output = render(context)
+        context.scopes.last[@to] = capture_output
+      end
       output
     end
 

lib/liquid/tags/case.rb

@@ -19,18 +19,13 @@ module Liquid
     end
 
     def parse(tokens)
-      body = BlockBody.new
+      body = new_body
       body = @blocks.last.attachment while parse_body(body, tokens)
-      @blank = @blocks.all? { |condition| condition.attachment.blank? }
-      if @blank
+      if blank?
         @blocks.each { |condition| condition.attachment.remove_blank_strings }
       end
     end
 
-    def blank?
-      @blank
-    end
-
     def nodelist
       @blocks.map(&:attachment)
     end
@@ -64,7 +59,7 @@ module Liquid
     private
 
     def record_when_condition(markup)
-      body = BlockBody.new
+      body = new_body
 
       while markup
         unless markup =~ WhenSyntax
@@ -85,7 +80,7 @@ module Liquid
       end
 
       block = ElseCondition.new
-      block.attach(BlockBody.new)
+      block.attach(new_body)
       @blocks << block
     end
 

lib/liquid/tags/for.rb

@@ -54,7 +54,7 @@ module Liquid
       super
       @from = @limit = nil
       parse_with_selected_parser(markup)
-      @for_block = BlockBody.new
+      @for_block = new_body
       @else_block = nil
     end
 
@@ -62,24 +62,19 @@ module Liquid
       if parse_body(@for_block, tokens)
         parse_body(@else_block, tokens)
       end
-      @blank = @for_block.blank? && (@else_block.nil? || @else_block.blank?)
-      if @blank
+      if blank?
         @for_block.remove_blank_strings
         @else_block&.remove_blank_strings
       end
     end
 
-    def blank?
-      @blank
-    end
-
     def nodelist
       @else_block ? [@for_block, @else_block] : [@for_block]
     end
 
     def unknown_tag(tag, markup, tokens)
       return super unless tag == 'else'
-      @else_block = BlockBody.new
+      @else_block = new_body
     end
 
     def render_to_output_buffer(context, output)

lib/liquid/tags/if.rb

@@ -31,16 +31,11 @@ module Liquid
     def parse(tokens)
       while parse_body(@blocks.last.attachment, tokens)
       end
-      @blank = @blocks.all? { |condition| condition.attachment.blank? }
-      if @blank
+      if blank?
         @blocks.each { |condition| condition.attachment.remove_blank_strings }
       end
     end
 
-    def blank?
-      @blank
-    end
-
     def unknown_tag(tag, markup, tokens)
       if ['elsif', 'else'].include?(tag)
         push_block(tag, markup)
@@ -69,7 +64,7 @@ module Liquid
       end
 
       @blocks.push(block)
-      block.attach(BlockBody.new)
+      block.attach(new_body)
     end
 
     def lax_parse(markup)

test/integration/template_test.rb

@@ -111,13 +111,12 @@ class TemplateTest < Minitest::Test
 
   def test_resource_limits_render_length
     t = Template.parse("0123456789")
-    t.resource_limits.render_length_limit = 5
+    t.resource_limits.render_length_limit = 9
     assert_equal("Liquid error: Memory limits exceeded", t.render)
     assert(t.resource_limits.reached?)
 
     t.resource_limits.render_length_limit = 10
     assert_equal("0123456789", t.render!)
-    refute_nil(t.resource_limits.render_length)
   end
 
   def test_resource_limits_render_score
@@ -180,36 +179,33 @@ class TemplateTest < Minitest::Test
     t.render!
     assert(t.resource_limits.assign_score > 0)
     assert(t.resource_limits.render_score > 0)
-    assert(t.resource_limits.render_length > 0)
   end
 
   def test_render_length_persists_between_blocks
     t = Template.parse("{% if true %}aaaa{% endif %}")
-    t.resource_limits.render_length_limit = 7
+    t.resource_limits.render_length_limit = 3
     assert_equal("Liquid error: Memory limits exceeded", t.render)
-    t.resource_limits.render_length_limit = 8
+    t.resource_limits.render_length_limit = 4
     assert_equal("aaaa", t.render)
 
     t = Template.parse("{% if true %}aaaa{% endif %}{% if true %}bbb{% endif %}")
-    t.resource_limits.render_length_limit = 13
+    t.resource_limits.render_length_limit = 6
     assert_equal("Liquid error: Memory limits exceeded", t.render)
-    t.resource_limits.render_length_limit = 14
+    t.resource_limits.render_length_limit = 7
     assert_equal("aaaabbb", t.render)
 
     t = Template.parse("{% if true %}a{% endif %}{% if true %}b{% endif %}{% if true %}a{% endif %}{% if true %}b{% endif %}{% if true %}a{% endif %}{% if true %}b{% endif %}")
     t.resource_limits.render_length_limit = 5
     assert_equal("Liquid error: Memory limits exceeded", t.render)
-    t.resource_limits.render_length_limit = 11
-    assert_equal("Liquid error: Memory limits exceeded", t.render)
-    t.resource_limits.render_length_limit = 12
+    t.resource_limits.render_length_limit = 6
     assert_equal("ababab", t.render)
   end
 
   def test_render_length_uses_number_of_bytes_not_characters
     t = Template.parse("{% if true %}すごい{% endif %}")
-    t.resource_limits.render_length_limit = 10
+    t.resource_limits.render_length_limit = 8
     assert_equal("Liquid error: Memory limits exceeded", t.render)
-    t.resource_limits.render_length_limit = 18
+    t.resource_limits.render_length_limit = 9
     assert_equal("すごい", t.render)
   end
 
@@ -219,7 +215,6 @@ class TemplateTest < Minitest::Test
     t.render!(context)
     assert(context.resource_limits.assign_score > 0)
     assert(context.resource_limits.render_score > 0)
-    assert(context.resource_limits.render_length > 0)
   end
 
   def test_can_use_drop_as_context

test/integration/trim_mode_test.rb

@@ -528,4 +528,21 @@ class TrimModeTest < Minitest::Test
     END_EXPECTED
     assert_template_result(expected, text)
   end
+
+  def test_bug_compatible_pre_trim
+    template = Liquid::Template.parse("\n {%- raw %}{% endraw %}", bug_compatible_whitespace_trimming: true)
+    assert_equal("\n", template.render)
+
+    template = Liquid::Template.parse("\n {%- if true %}{% endif %}", bug_compatible_whitespace_trimming: true)
+    assert_equal("\n", template.render)
+
+    template = Liquid::Template.parse("{{ 'B' }} \n{%- if true %}C{% endif %}", bug_compatible_whitespace_trimming: true)
+    assert_equal("B C", template.render)
+
+    template = Liquid::Template.parse("B\n {%- raw %}{% endraw %}", bug_compatible_whitespace_trimming: true)
+    assert_equal("B", template.render)
+
+    template = Liquid::Template.parse("B\n {%- if true %}{% endif %}", bug_compatible_whitespace_trimming: true)
+    assert_equal("B", template.render)
+  end
 end # TrimModeTest