Port liquid-c bug compatible whitespace trimming

The parse_context no longer needs to be passed in because it is available
through through an attr_reader on the instance. However, the markup and
tokenizer weren't made available.  This refactor also makes the parameters
given to Document#unknown_tag consistent with Block#unknown_tag.

lib/liquid/block.rb

@@ -59,7 +59,7 @@ module Liquid
 
     # @api public
     def new_body
-      parse_context.new_block_body
+      BlockBody.new
     end
 
     # @api public

lib/liquid/block_body.rb

@@ -71,24 +71,13 @@ module Liquid
     # @api private
     def self.render_node(context, output, node)
       node.render_to_output_buffer(context, output)
-    rescue => exc
-      blank_tag = !node.instance_of?(Variable) && node.blank?
-      rescue_render_node(context, output, node.line_number, exc, blank_tag)
-    end
-
-    # @api private
-    def self.rescue_render_node(context, output, line_number, exc, blank_tag)
-      case exc
-      when MemoryError
-        raise
-      when UndefinedVariable, UndefinedDropMethod, UndefinedFilter
-        context.handle_error(exc, line_number)
-      else
-        error_message = context.handle_error(exc, line_number)
-        unless blank_tag # conditional for backwards compatibility
-          output << error_message
-        end
-      end
+    rescue UndefinedVariable, UndefinedDropMethod, UndefinedFilter => e
+      context.handle_error(e, node.line_number)
+    rescue MemoryError
+      raise
+    rescue ::StandardError => e
+      line_number = node.is_a?(String) ? nil : node.line_number
+      output << context.handle_error(e, line_number)
     end
 
     private def parse_liquid_tag(markup, parse_context)
@@ -153,7 +142,11 @@ module Liquid
       if token[2] == WhitespaceControl
         previous_token = @nodelist.last
         if previous_token.is_a?(String)
+          first_byte = previous_token.getbyte(0)
           previous_token.rstrip!
+          if previous_token.empty? && parse_context[:bug_compatible_whitespace_trimming] && first_byte
+            previous_token << first_byte
+          end
         end
       end
       parse_context.trim_whitespace = (token[-3] == WhitespaceControl)

lib/liquid/condition.rb

@@ -27,32 +27,10 @@ module Liquid
       end,
     }
 
-    class MethodLiteral
-      attr_reader :method_name, :to_s
-
-      def initialize(method_name, to_s)
-        @method_name = method_name
-        @to_s = to_s
-      end
-
-      def to_liquid
-        to_s
-      end
-    end
-
-    @@method_literals = {
-      'blank' => MethodLiteral.new(:blank?, '').freeze,
-      'empty' => MethodLiteral.new(:empty?, '').freeze,
-    }
-
     def self.operators
       @@operators
     end
 
-    def self.parse_expression(markup)
-      @@method_literals[markup] || Expression.parse(markup)
-    end
-
     attr_reader :attachment, :child_condition
     attr_accessor :left, :operator, :right
 
@@ -113,7 +91,7 @@ module Liquid
     private
 
     def equal_variables(left, right)
-      if left.is_a?(MethodLiteral)
+      if left.is_a?(Liquid::Expression::MethodLiteral)
         if right.respond_to?(left.method_name)
           return right.send(left.method_name)
         else
@@ -121,7 +99,7 @@ module Liquid
         end
       end
 
-      if right.is_a?(MethodLiteral)
+      if right.is_a?(Liquid::Expression::MethodLiteral)
         if left.respond_to?(right.method_name)
           return left.send(right.method_name)
         else

lib/liquid/document.rb

@@ -47,7 +47,7 @@ module Liquid
     private
 
     def new_body
-      parse_context.new_block_body
+      Liquid::BlockBody.new
     end
 
     def parse_body(tokenizer)

lib/liquid/expression.rb

@@ -2,12 +2,25 @@
 
 module Liquid
   class Expression
+    class MethodLiteral
+      attr_reader :method_name, :to_s
+
+      def initialize(method_name, to_s)
+        @method_name = method_name
+        @to_s = to_s
+      end
+
+      def to_liquid
+        to_s
+      end
+    end
+
     LITERALS = {
       nil => nil, 'nil' => nil, 'null' => nil, '' => nil,
       'true' => true,
       'false' => false,
-      'blank' => '',
-      'empty' => ''
+      'blank' => MethodLiteral.new(:blank?, '').freeze,
+      'empty' => MethodLiteral.new(:empty?, '').freeze
     }.freeze
 
     SINGLE_QUOTED_STRING = /\A'(.*)'\z/m

lib/liquid/parse_context.rb

@@ -19,10 +19,6 @@ module Liquid
       @options[option_key]
     end
 
-    def new_block_body
-      Liquid::BlockBody.new
-    end
-
     def partial=(value)
       @partial = value
       @options = value ? partial_options : @template_options

lib/liquid/tags/case.rb

@@ -68,7 +68,7 @@ module Liquid
 
         markup = Regexp.last_match(2)
 
-        block = Condition.new(@left, '==', Condition.parse_expression(Regexp.last_match(1)))
+        block = Condition.new(@left, '==', Expression.parse(Regexp.last_match(1)))
         block.attach(body)
         @blocks << block
       end

lib/liquid/tags/if.rb

@@ -67,22 +67,18 @@ module Liquid
       block.attach(new_body)
     end
 
-    def parse_expression(markup)
-      Condition.parse_expression(markup)
-    end
-
     def lax_parse(markup)
       expressions = markup.scan(ExpressionsAndOperators)
       raise SyntaxError, options[:locale].t("errors.syntax.if") unless expressions.pop =~ Syntax
 
-      condition = Condition.new(parse_expression(Regexp.last_match(1)), Regexp.last_match(2), parse_expression(Regexp.last_match(3)))
+      condition = Condition.new(Expression.parse(Regexp.last_match(1)), Regexp.last_match(2), Expression.parse(Regexp.last_match(3)))
 
       until expressions.empty?
         operator = expressions.pop.to_s.strip
 
         raise SyntaxError, options[:locale].t("errors.syntax.if") unless expressions.pop.to_s =~ Syntax
 
-        new_condition = Condition.new(parse_expression(Regexp.last_match(1)), Regexp.last_match(2), parse_expression(Regexp.last_match(3)))
+        new_condition = Condition.new(Expression.parse(Regexp.last_match(1)), Regexp.last_match(2), Expression.parse(Regexp.last_match(3)))
         raise SyntaxError, options[:locale].t("errors.syntax.if") unless BOOLEAN_OPERATORS.include?(operator)
         new_condition.send(operator, condition)
         condition = new_condition
@@ -110,9 +106,9 @@ module Liquid
     end
 
     def parse_comparison(p)
-      a = parse_expression(p.expression)
+      a = Expression.parse(p.expression)
       if (op = p.consume?(:comparison))
-        b = parse_expression(p.expression)
+        b = Expression.parse(p.expression)
         Condition.new(a, op, b)
       else
         Condition.new(a)

test/integration/block_test.rb

@@ -11,48 +11,4 @@ class BlockTest < Minitest::Test
     end
     assert_equal(exc.message, "Liquid syntax error: 'endunless' is not a valid delimiter for if tags. use endif")
   end
-
-  def test_with_custom_tag
-    with_custom_tag('testtag', Block) do
-      assert Liquid::Template.parse("{% testtag %} {% endtesttag %}")
-    end
-  end
-
-  def test_custom_block_tags_have_a_default_render_to_output_buffer_method_for_backwards_compatibility
-    klass1 = Class.new(Block) do
-      def render(*)
-        'hello'
-      end
-    end
-
-    with_custom_tag('blabla', klass1) do
-      template = Liquid::Template.parse("{% blabla %} bla {% endblabla %}")
-
-      assert_equal 'hello', template.render
-
-      buf    = +''
-      output = template.render({}, output: buf)
-      assert_equal 'hello', output
-      assert_equal 'hello', buf
-      assert_equal buf.object_id, output.object_id
-    end
-
-    klass2 = Class.new(klass1) do
-      def render(*)
-        'foo' + super + 'bar'
-      end
-    end
-
-    with_custom_tag('blabla', klass2) do
-      template = Liquid::Template.parse("{% blabla %} foo {% endblabla %}")
-
-      assert_equal 'foohellobar', template.render
-
-      buf    = +''
-      output = template.render({}, output: buf)
-      assert_equal 'foohellobar', output
-      assert_equal 'foohellobar', buf
-      assert_equal buf.object_id, output.object_id
-    end
-  end
 end

test/integration/error_handling_test.rb

@@ -261,12 +261,4 @@ class ErrorHandlingTest < Minitest::Test
     assert_equal("Argument error:\nLiquid error (product line 1): argument error", page)
     assert_equal("product", template.errors.first.template_name)
   end
-
-  def test_bug_compatible_silencing_of_errors_in_blank_nodes
-    output = Liquid::Template.parse("{% assign x = 0 %}{% if 1 < '2' %}not blank{% assign x = 3 %}{% endif %}{{ x }}").render
-    assert_equal("Liquid error: comparison of Integer with String failed0", output)
-
-    output = Liquid::Template.parse("{% assign x = 0 %}{% if 1 < '2' %}{% assign x = 3 %}{% endif %}{{ x }}").render
-    assert_equal("0", output)
-  end
 end

test/integration/security_test.rb

@@ -43,22 +43,15 @@ class SecurityTest < Minitest::Test
     assert_equal(expected, Template.parse(text).render!(@assigns, filters: SecurityFilter))
   end
 
-  def test_does_not_permanently_add_filters_to_symbol_table
+  def test_does_not_add_filters_to_symbol_table
     current_symbols = Symbol.all_symbols
 
-    # MRI imprecisely marks objects found on the C stack, which can result
-    # in uninitialized memory being marked. This can even result in the test failing
-    # deterministically for a given compilation of ruby. Using a separate thread will
-    # keep these writes of the symbol pointer on a separate stack that will be garbage
-    # collected after Thread#join.
-    Thread.new do
-      test = %( {{ "some_string" | a_bad_filter }} )
-      Template.parse(test).render!
-      nil
-    end.join
+    test = %( {{ "some_string" | a_bad_filter }} )
 
-    GC.start
+    template = Template.parse(test)
+    assert_equal([], (Symbol.all_symbols - current_symbols))
 
+    template.render!
     assert_equal([], (Symbol.all_symbols - current_symbols))
   end
 

test/integration/tag_test.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require 'test_helper'
-
-class TagTest < Minitest::Test
-  include Liquid
-
-  def test_custom_tags_have_a_default_render_to_output_buffer_method_for_backwards_compatibility
-    klass1 = Class.new(Tag) do
-      def render(*)
-        'hello'
-      end
-    end
-
-    with_custom_tag('blabla', klass1) do
-      template = Liquid::Template.parse("{% blabla %}")
-
-      assert_equal 'hello', template.render
-
-      buf    = +''
-      output = template.render({}, output: buf)
-      assert_equal 'hello', output
-      assert_equal 'hello', buf
-      assert_equal buf.object_id, output.object_id
-    end
-
-    klass2 = Class.new(klass1) do
-      def render(*)
-        'foo' + super + 'bar'
-      end
-    end
-
-    with_custom_tag('blabla', klass2) do
-      template = Liquid::Template.parse("{% blabla %}")
-
-      assert_equal 'foohellobar', template.render
-
-      buf    = +''
-      output = template.render({}, output: buf)
-      assert_equal 'foohellobar', output
-      assert_equal 'foohellobar', buf
-      assert_equal buf.object_id, output.object_id
-    end
-  end
-end

test/integration/trim_mode_test.rb

@@ -528,4 +528,21 @@ class TrimModeTest < Minitest::Test
     END_EXPECTED
     assert_template_result(expected, text)
   end
+
+  def test_bug_compatible_pre_trim
+    template = Liquid::Template.parse("\n {%- raw %}{% endraw %}", bug_compatible_whitespace_trimming: true)
+    assert_equal("\n", template.render)
+
+    template = Liquid::Template.parse("\n {%- if true %}{% endif %}", bug_compatible_whitespace_trimming: true)
+    assert_equal("\n", template.render)
+
+    template = Liquid::Template.parse("{{ 'B' }} \n{%- if true %}C{% endif %}", bug_compatible_whitespace_trimming: true)
+    assert_equal("B C", template.render)
+
+    template = Liquid::Template.parse("B\n {%- raw %}{% endraw %}", bug_compatible_whitespace_trimming: true)
+    assert_equal("B", template.render)
+
+    template = Liquid::Template.parse("B\n {%- if true %}{% endif %}", bug_compatible_whitespace_trimming: true)
+    assert_equal("B", template.render)
+  end
 end # TrimModeTest

test/unit/block_unit_test.rb

@@ -45,9 +45,53 @@ class BlockUnitTest < Minitest::Test
     assert_equal(3, template.root.nodelist.size)
   end
 
+  def test_with_custom_tag
+    with_custom_tag('testtag', Block) do
+      assert Liquid::Template.parse("{% testtag %} {% endtesttag %}")
+    end
+  end
+
+  def test_custom_block_tags_have_a_default_render_to_output_buffer_method_for_backwards_compatibility
+    klass1 = Class.new(Block) do
+      def render(*)
+        'hello'
+      end
+    end
+
+    with_custom_tag('blabla', klass1) do
+      template = Liquid::Template.parse("{% blabla %} bla {% endblabla %}")
+
+      assert_equal 'hello', template.render
+
+      buf    = +''
+      output = template.render({}, output: buf)
+      assert_equal 'hello', output
+      assert_equal 'hello', buf
+      assert_equal buf.object_id, output.object_id
+    end
+
+    klass2 = Class.new(klass1) do
+      def render(*)
+        'foo' + super + 'bar'
+      end
+    end
+
+    with_custom_tag('blabla', klass2) do
+      template = Liquid::Template.parse("{% blabla %} foo {% endblabla %}")
+
+      assert_equal 'foohellobar', template.render
+
+      buf    = +''
+      output = template.render({}, output: buf)
+      assert_equal 'foohellobar', output
+      assert_equal 'foohellobar', buf
+      assert_equal buf.object_id, output.object_id
+    end
+  end
+
   private
 
   def block_types(nodelist)
     nodelist.collect(&:class)
   end
-end
+end # VariableTest

test/unit/context_unit_test.rb

@@ -65,7 +65,7 @@ class ArrayLike
   end
 end
 
-class ContextTest < Minitest::Test
+class ContextUnitTest < Minitest::Test
   include Liquid
 
   def setup

test/unit/tag_unit_test.rb

@@ -20,4 +20,42 @@ class TagUnitTest < Minitest::Test
     tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal('some_tag', tag.tag_name)
   end
+
+  def test_custom_tags_have_a_default_render_to_output_buffer_method_for_backwards_compatibility
+    klass1 = Class.new(Tag) do
+      def render(*)
+        'hello'
+      end
+    end
+
+    with_custom_tag('blabla', klass1) do
+      template = Liquid::Template.parse("{% blabla %}")
+
+      assert_equal 'hello', template.render
+
+      buf    = +''
+      output = template.render({}, output: buf)
+      assert_equal 'hello', output
+      assert_equal 'hello', buf
+      assert_equal buf.object_id, output.object_id
+    end
+
+    klass2 = Class.new(klass1) do
+      def render(*)
+        'foo' + super + 'bar'
+      end
+    end
+
+    with_custom_tag('blabla', klass2) do
+      template = Liquid::Template.parse("{% blabla %}")
+
+      assert_equal 'foohellobar', template.render
+
+      buf    = +''
+      output = template.render({}, output: buf)
+      assert_equal 'foohellobar', output
+      assert_equal 'foohellobar', buf
+      assert_equal buf.object_id, output.object_id
+    end
+  end
 end