refactor: Reduce maximum block nesting in Liquid::BlockBody#parse

Avoid hash with default values due to inconsistent marshaling

.rubocop.yml

@@ -7,8 +7,6 @@ AllCops:
 
 Metrics/BlockNesting:
   Max: 3
-  Exclude:
-    - 'lib/liquid/block_body.rb'
 
 Metrics/ModuleLength:
   Enabled: false

.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 
 rvm:
-  - 2.0
   - 2.1
   - 2.2
+  - 2.3.3
   - ruby-head
   - jruby-head
 #  - rbx-2
@@ -19,6 +19,10 @@ matrix:
   allow_failures:
     - rvm: jruby-head
 
+install:
+  - gem install rainbow -v 2.2.1
+  - bundle install
+
 script: "bundle exec rake"
 
 notifications:

Gemfile

@@ -1,11 +1,14 @@
 source 'https://rubygems.org'
 
 gemspec
-gem 'stackprof', platforms: :mri_21
+
+gem 'stackprof', platforms: :mri
+
+group :benchmark, :test do
+  gem 'benchmark-ips'
+end
 
 group :test do
-  gem 'spy', '0.4.1'
-  gem 'benchmark-ips'
   gem 'rubocop', '0.34.2'
 
   platform :mri do

History.md

@@ -1,8 +1,10 @@
 # Liquid Change Log
 
-## 4.0.0 / not yet released / branch "master"
+## 4.0.0 / 2016-12-14 / branch "4-0-stable"
 
 ### Changed
+* Render an opaque internal error by default for non-Liquid::Error (#835) [Dylan Thacker-Smith]
+* Ruby 2.0 support dropped (#832) [Dylan Thacker-Smith]
 * Add to_number Drop method to allow custom drops to work with number filters (#731)
 * Add strict_variables and strict_filters options to detect undefined references (#691)
 * Improve loop performance (#681) [Florian Weingarten]
@@ -18,10 +20,13 @@
 * Add concat filter to concatenate arrays (#429) [Diogo Beato]
 * Ruby 1.9 support dropped (#491) [Justin Li]
 * Liquid::Template.file_system's read_template_file method is no longer passed the context. (#441) [James Reid-Smith]
-* Remove support for `liquid_methods`
+* Remove `liquid_methods` (See https://github.com/Shopify/liquid/pull/568 for replacement)
 * Liquid::Template.register_filter raises when the module overrides registered public methods as private or protected (#705) [Gaurav Chande]
 
 ### Fixed
+
+* Fix variable names being detected as an operator when starting with contains (#788) [Michael Angell]
+* Fix include tag used with strict_variables (#828) [QuickPay]
 * Fix map filter when value is a Proc (#672) [Guillaume Malette]
 * Fix truncate filter when value is not a string (#672) [Guillaume Malette]
 * Fix behaviour of escape filter when input is nil (#665) [Tanel Jakobsoo]

circle.yml

@@ -1,3 +1,3 @@
 machine:
   ruby:
-    version: ruby-2.0
+    version: ruby-2.1

lib/liquid/block.rb

@@ -1,5 +1,7 @@
 module Liquid
   class Block < Tag
+    MAX_DEPTH = 100
+
     def initialize(tag_name, markup, options)
       super
       @blank = true
@@ -48,17 +50,25 @@ module Liquid
     protected
 
     def parse_body(body, tokens)
-      body.parse(tokens, parse_context) do |end_tag_name, end_tag_params|
-        @blank &&= body.blank?
+      if parse_context.depth >= MAX_DEPTH
+        raise StackLevelError, "Nesting too deep".freeze
+      end
+      parse_context.depth += 1
+      begin
+        body.parse(tokens, parse_context) do |end_tag_name, end_tag_params|
+          @blank &&= body.blank?
 
-        return false if end_tag_name == block_delimiter
-        unless end_tag_name
-          raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_never_closed".freeze, block_name: block_name))
+          return false if end_tag_name == block_delimiter
+          unless end_tag_name
+            raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_never_closed".freeze, block_name: block_name))
+          end
+
+          # this tag is not registered with the system
+          # pass it to the current block for special handling or error reporting
+          unknown_tag(end_tag_name, end_tag_params, tokens)
         end
-
-        # this tag is not registered with the system
-        # pass it to the current block for special handling or error reporting
-        unknown_tag(end_tag_name, end_tag_params, tokens)
+      ensure
+        parse_context.depth -= 1
       end
 
       true

lib/liquid/block_body.rb

@@ -15,38 +15,35 @@ module Liquid
     def parse(tokenizer, parse_context)
       parse_context.line_number = tokenizer.line_number
       while token = tokenizer.shift
-        unless token.empty?
-          case
-          when token.start_with?(TAGSTART)
-            whitespace_handler(token, parse_context)
-            if token =~ FullToken
-              tag_name = $1
-              markup = $2
-              # fetch the tag from registered blocks
-              if tag = registered_tags[tag_name]
-                new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
-                @blank &&= new_tag.blank?
-                @nodelist << new_tag
-              else
-                # end parsing if we reach an unknown tag and let the caller decide
-                # determine how to proceed
-                return yield tag_name, markup
-              end
-            else
-              raise_missing_tag_terminator(token, parse_context)
-            end
-          when token.start_with?(VARSTART)
-            whitespace_handler(token, parse_context)
-            @nodelist << create_variable(token, parse_context)
-            @blank = false
-          else
-            if parse_context.trim_whitespace
-              token.lstrip!
-            end
-            parse_context.trim_whitespace = false
-            @nodelist << token
-            @blank &&= !!(token =~ /\A\s*\z/)
+        next if token.empty?
+        case
+        when token.start_with?(TAGSTART)
+          whitespace_handler(token, parse_context)
+          unless token =~ FullToken
+            raise_missing_tag_terminator(token, parse_context)
           end
+          tag_name = $1
+          markup = $2
+          # fetch the tag from registered blocks
+          unless tag = registered_tags[tag_name]
+            # end parsing if we reach an unknown tag and let the caller decide
+            # determine how to proceed
+            return yield tag_name, markup
+          end
+          new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
+          @blank &&= new_tag.blank?
+          @nodelist << new_tag
+        when token.start_with?(VARSTART)
+          whitespace_handler(token, parse_context)
+          @nodelist << create_variable(token, parse_context)
+          @blank = false
+        else
+          if parse_context.trim_whitespace
+            token.lstrip!
+          end
+          parse_context.trim_whitespace = false
+          @nodelist << token
+          @blank &&= !!(token =~ /\A\s*\z/)
         end
         parse_context.line_number = tokenizer.line_number
       end
@@ -96,7 +93,8 @@ module Liquid
           context.handle_error(e, token.line_number)
           output << nil
         rescue ::StandardError => e
-          output << context.handle_error(e, token.line_number)
+          line_number = token.is_a?(String) ? nil : token.line_number
+          output << context.handle_error(e, line_number)
         end
       end
 
@@ -106,7 +104,7 @@ module Liquid
     private
 
     def render_node(node, context)
-      node_output = (node.respond_to?(:render) ? node.render(context) : node)
+      node_output = node.is_a?(String) ? node : node.render(context)
       node_output = node_output.is_a?(Array) ? node_output.join : node_output.to_s
 
       context.resource_limits.render_length += node_output.length

lib/liquid/condition.rb

@@ -41,16 +41,22 @@ module Liquid
     end
 
     def evaluate(context = Context.new)
-      result = interpret_condition(left, right, operator, context)
+      condition = self
+      result = nil
+      loop do
+        result = interpret_condition(condition.left, condition.right, condition.operator, context)
 
-      case @child_relation
-      when :or
-        result || @child_condition.evaluate(context)
-      when :and
-        result && @child_condition.evaluate(context)
-      else
-        result
+        case condition.child_relation
+        when :or
+          break if result
+        when :and
+          break unless result
+        else
+          break
+        end
+        condition = condition.child_condition
       end
+      result
     end
 
     def or(condition)
@@ -75,6 +81,10 @@ module Liquid
       "#<Condition #{[@left, @operator, @right].compact.join(' '.freeze)}>"
     end
 
+    protected
+
+    attr_reader :child_relation, :child_condition
+
     private
 
     def equal_variables(left, right)
@@ -110,7 +120,7 @@ module Liquid
 
       if operation.respond_to?(:call)
         operation.call(self, left, right)
-      elsif left.respond_to?(operation) && right.respond_to?(operation)
+      elsif left.respond_to?(operation) && right.respond_to?(operation) && !left.is_a?(Hash) && !right.is_a?(Hash)
         begin
           left.send(operation, right)
         rescue ::ArgumentError => e

lib/liquid/context.rb

@@ -13,7 +13,7 @@ module Liquid
   #   context['bob']  #=> nil  class Context
   class Context
     attr_reader :scopes, :errors, :registers, :environments, :resource_limits
-    attr_accessor :exception_handler, :template_name, :partial, :global_filter, :strict_variables, :strict_filters
+    attr_accessor :exception_renderer, :template_name, :partial, :global_filter, :strict_variables, :strict_filters
 
     def initialize(environments = {}, outer_scope = {}, registers = {}, rethrow_errors = false, resource_limits = nil)
       @environments     = [environments].flatten
@@ -27,8 +27,9 @@ module Liquid
 
       @this_stack_used = false
 
+      self.exception_renderer = Template.default_exception_renderer
       if rethrow_errors
-        self.exception_handler = ->(e) { raise }
+        self.exception_renderer = ->(e) { raise }
       end
 
       @interrupts = []
@@ -74,30 +75,11 @@ module Liquid
     end
 
     def handle_error(e, line_number = nil)
-      if e.is_a?(Liquid::Error)
-        e.template_name ||= template_name
-        e.line_number ||= line_number
-      end
-
-      output = nil
-
-      if exception_handler
-        result = exception_handler.call(e)
-        case result
-        when Exception
-          e = result
-          if e.is_a?(Liquid::Error)
-            e.template_name ||= template_name
-            e.line_number ||= line_number
-          end
-        when String
-          output = result
-        else
-          raise if result
-        end
-      end
+      e = internal_error unless e.is_a?(Liquid::Error)
+      e.template_name ||= template_name
+      e.line_number ||= line_number
       errors.push(e)
-      output || Liquid::Error.render(e)
+      exception_renderer.call(e).to_s
     end
 
     def invoke(method, *args)
@@ -107,7 +89,7 @@ module Liquid
     # Push new local scope on the stack. use <tt>Context#stack</tt> instead
     def push(new_scope = {})
       @scopes.unshift(new_scope)
-      raise StackLevelError, "Nesting too deep".freeze if @scopes.length > 100
+      raise StackLevelError, "Nesting too deep".freeze if @scopes.length > Block::MAX_DEPTH
     end
 
     # Merge a hash of variables in the current local scope
@@ -178,7 +160,7 @@ module Liquid
     end
 
     # Fetches an object starting at the local scope and then moving up the hierachy
-    def find_variable(key)
+    def find_variable(key, raise_on_not_found: true)
       # This was changed from find() to find_index() because this is a very hot
       # path and find_index() is optimized in MRI to reduce object allocation
       index = @scopes.find_index { |s| s.key?(key) }
@@ -188,8 +170,10 @@ module Liquid
 
       if scope.nil?
         @environments.each do |e|
-          variable = lookup_and_evaluate(e, key)
-          unless variable.nil?
+          variable = lookup_and_evaluate(e, key, raise_on_not_found: raise_on_not_found)
+          # When lookup returned a value OR there is no value but the lookup also did not raise
+          # then it is the value we are looking for.
+          if !variable.nil? || @strict_variables && raise_on_not_found
             scope = e
             break
           end
@@ -197,7 +181,7 @@ module Liquid
       end
 
       scope ||= @environments.last || @scopes.last
-      variable ||= lookup_and_evaluate(scope, key)
+      variable ||= lookup_and_evaluate(scope, key, raise_on_not_found: raise_on_not_found)
 
       variable = variable.to_liquid
       variable.context = self if variable.respond_to?(:context=)
@@ -205,8 +189,8 @@ module Liquid
       variable
     end
 
-    def lookup_and_evaluate(obj, key)
-      if @strict_variables && obj.respond_to?(:key?) && !obj.key?(key)
+    def lookup_and_evaluate(obj, key, raise_on_not_found: true)
+      if @strict_variables && raise_on_not_found && obj.respond_to?(:key?) && !obj.key?(key)
         raise Liquid::UndefinedVariable, "undefined variable #{key}"
       end
 
@@ -221,6 +205,13 @@ module Liquid
 
     private
 
+    def internal_error
+      # raise and catch to set backtrace and cause on exception
+      raise Liquid::InternalError, 'internal'
+    rescue Liquid::InternalError => exc
+      exc
+    end
+
     def squash_instance_assigns_with_environments
       @scopes.last.each_key do |k|
         @environments.each do |env|

lib/liquid/errors.rb

@@ -17,14 +17,6 @@ module Liquid
       str
     end
 
-    def self.render(e)
-      if e.is_a?(Liquid::Error)
-        e.to_s
-      else
-        "Liquid error: #{e}"
-      end
-    end
-
     private
 
     def message_prefix
@@ -60,4 +52,5 @@ module Liquid
   UndefinedDropMethod = Class.new(Error)
   UndefinedFilter = Class.new(Error)
   MethodOverrideError = Class.new(Error)
+  InternalError = Class.new(Error)
 end

lib/liquid/extensions.rb

@@ -7,6 +7,12 @@ class String # :nodoc:
   end
 end
 
+class Symbol # :nodoc:
+  def to_liquid
+    to_s
+  end
+end
+
 class Array # :nodoc:
   def to_liquid
     self

lib/liquid/lexer.rb

@@ -18,10 +18,10 @@ module Liquid
     DOUBLE_STRING_LITERAL = /"[^\"]*"/
     NUMBER_LITERAL = /-?\d+(\.\d+)?/
     DOTDOT = /\.\./
-    COMPARISON_OPERATOR = /==|!=|<>|<=?|>=?|contains/
+    COMPARISON_OPERATOR = /==|!=|<>|<=?|>=?|contains(?=\s)/
 
     def initialize(input)
-      @ss = StringScanner.new(input.rstrip)
+      @ss = StringScanner.new(input)
     end
 
     def tokenize
@@ -29,6 +29,7 @@ module Liquid
 
       until @ss.eos?
         @ss.skip(/\s*/)
+        break if @ss.eos?
         tok = case
         when t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t]
         when t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t]

lib/liquid/parse_context.rb

@@ -1,12 +1,13 @@
 module Liquid
   class ParseContext
-    attr_accessor :locale, :line_number, :trim_whitespace
+    attr_accessor :locale, :line_number, :trim_whitespace, :depth
     attr_reader :partial, :warnings, :error_mode
 
     def initialize(options = {})
       @template_options = options ? options.dup : {}
       @locale = @template_options[:locale] ||= I18n.new
       @warnings = []
+      self.depth = 0
       self.partial = false
     end
 

lib/liquid/standardfilters.rb

@@ -33,7 +33,7 @@ module Liquid
     end
 
     def escape(input)
-      CGI.escapeHTML(input).untaint unless input.nil?
+      CGI.escapeHTML(input.to_s).untaint unless input.nil?
     end
     alias_method :h, :escape
 
@@ -42,11 +42,11 @@ module Liquid
     end
 
     def url_encode(input)
-      CGI.escape(input) unless input.nil?
+      CGI.escape(input.to_s) unless input.nil?
     end
 
     def url_decode(input)
-      CGI.unescape(input) unless input.nil?
+      CGI.unescape(input.to_s) unless input.nil?
     end
 
     def slice(input, offset, length = nil)
@@ -65,9 +65,10 @@ module Liquid
       return if input.nil?
       input_str = input.to_s
       length = Utils.to_integer(length)
-      l = length - truncate_string.length
+      truncate_string_str = truncate_string.to_s
+      l = length - truncate_string_str.length
       l = 0 if l < 0
-      input_str.length > length ? input_str[0...l] + truncate_string : input_str
+      input_str.length > length ? input_str[0...l] + truncate_string_str : input_str
     end
 
     def truncatewords(input, words = 15, truncate_string = "...".freeze)
@@ -76,7 +77,7 @@ module Liquid
       words = Utils.to_integer(words)
       l = words - 1
       l = 0 if l < 0
-      wordlist.length > l ? wordlist[0..l].join(" ".freeze) + truncate_string : input
+      wordlist.length > l ? wordlist[0..l].join(" ".freeze) + truncate_string.to_s : input
     end
 
     # Split input string into an array of substrings separated by given pattern.
@@ -85,7 +86,7 @@ module Liquid
     #   <div class="summary">{{ post | split '//' | first }}</div>
     #
     def split(input, pattern)
-      input.to_s.split(pattern)
+      input.to_s.split(pattern.to_s)
     end
 
     def strip(input)
@@ -124,7 +125,15 @@ module Liquid
       elsif ary.empty? # The next two cases assume a non-empty array.
         []
       elsif ary.first.respond_to?(:[]) && !ary.first[property].nil?
-        ary.sort { |a, b| a[property] <=> b[property] }
+        ary.sort do |a, b|
+          a = a[property]
+          b = b[property]
+          if a && b
+            a <=> b
+          else
+            a ? -1 : 1
+          end
+        end
       end
     end
 
@@ -192,12 +201,14 @@ module Liquid
 
     # Replace occurrences of a string with another
     def replace(input, string, replacement = ''.freeze)
-      input.to_s.gsub(string.to_s, replacement.to_s)
+      replacement = replacement.to_s
+      input.to_s.gsub(string.to_s) { replacement }
     end
 
     # Replace the first occurrences of a string with another
     def replace_first(input, string, replacement = ''.freeze)
-      input.to_s.sub(string.to_s, replacement.to_s)
+      replacement = replacement.to_s
+      input.to_s.sub(string.to_s) { replacement }
     end
 
     # remove a substring
@@ -375,7 +386,7 @@ module Liquid
       end
 
       def join(glue)
-        to_a.join(glue)
+        to_a.join(glue.to_s)
       end
 
       def concat(args)

lib/liquid/strainer.rb

@@ -27,7 +27,7 @@ module Liquid
 
     def self.add_filter(filter)
       raise ArgumentError, "Expected module but got: #{filter.class}" unless filter.is_a?(Module)
-      unless self.class.include?(filter)
+      unless self.include?(filter)
         invokable_non_public_methods = (filter.private_instance_methods + filter.protected_instance_methods).select { |m| invokable?(m) }
         if invokable_non_public_methods.any?
           raise MethodOverrideError, "Filter overrides registered public methods as non public: #{invokable_non_public_methods.join(', ')}"
@@ -39,6 +39,7 @@ module Liquid
     end
 
     def self.global_filter(filter)
+      @@strainer_class_cache.clear
       @@global_strainer.add_filter(filter)
     end
 

lib/liquid/tags/cycle.rb

@@ -30,11 +30,11 @@ module Liquid
     end
 
     def render(context)
-      context.registers[:cycle] ||= Hash.new(0)
+      context.registers[:cycle] ||= {}
 
       context.stack do
         key = context.evaluate(@name)
-        iteration = context.registers[:cycle][key]
+        iteration = context.registers[:cycle][key].to_i
         result = context.evaluate(@variables[iteration])
         iteration += 1
         iteration  = 0 if iteration >= @variables.size

lib/liquid/tags/for.rb

@@ -23,7 +23,7 @@ module Liquid
   #      {{ item.name }}
   #    {% end %}
   #
-  #  To reverse the for loop simply use {% for item in collection reversed %}
+  #  To reverse the for loop simply use {% for item in collection reversed %} (note that the flag's spelling is different to the filter `reverse`)
   #
   # == Available variables:
   #
@@ -46,6 +46,9 @@ module Liquid
   class For < Block
     Syntax = /\A(#{VariableSegment}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o
 
+    attr_reader :collection_name
+    attr_reader :variable_name
+
     def initialize(tag_name, markup, options)
       super
       @from = @limit = nil
@@ -117,7 +120,7 @@ module Liquid
     private
 
     def collection_segment(context)
-      offsets = context.registers[:for] ||= Hash.new(0)
+      offsets = context.registers[:for] ||= {}
 
       from = if @from == :continue
         offsets[@name].to_i

lib/liquid/tags/if.rb

@@ -83,17 +83,20 @@ module Liquid
 
     def strict_parse(markup)
       p = Parser.new(markup)
-      condition = parse_binary_comparison(p)
+      condition = parse_binary_comparisons(p)
       p.consume(:end_of_string)
       condition
     end
 
-    def parse_binary_comparison(p)
+    def parse_binary_comparisons(p)
       condition = parse_comparison(p)
-      if op = (p.id?('and'.freeze) || p.id?('or'.freeze))
-        condition.send(op, parse_binary_comparison(p))
+      first_condition = condition
+      while op = (p.id?('and'.freeze) || p.id?('or'.freeze))
+        child_condition = parse_comparison(p)
+        condition.send(op, child_condition)
+        condition = child_condition
       end
-      condition
+      first_condition
     end
 
     def parse_comparison(p)

lib/liquid/tags/include.rb

@@ -50,7 +50,7 @@ module Liquid
       variable = if @variable_name_expr
         context.evaluate(@variable_name_expr)
       else
-        context.find_variable(template_name)
+        context.find_variable(template_name, raise_on_not_found: false)
       end
 
       old_template_name = context.template_name

lib/liquid/template.rb

@@ -69,6 +69,11 @@ module Liquid
       # :error raises an error when tainted output is used
       attr_writer :taint_mode
 
+      attr_accessor :default_exception_renderer
+      Template.default_exception_renderer = lambda do |exception|
+        exception
+      end
+
       def file_system
         @@file_system
       end
@@ -167,7 +172,7 @@ module Liquid
         c = args.shift
 
         if @rethrow_errors
-          c.exception_handler = ->(e) { raise }
+          c.exception_renderer = ->(e) { raise }
         end
 
         c
@@ -241,7 +246,7 @@ module Liquid
     def apply_options_to_context(context, options)
       context.add_filters(options[:filters]) if options[:filters]
       context.global_filter = options[:global_filter] if options[:global_filter]
-      context.exception_handler = options[:exception_handler] if options[:exception_handler]
+      context.exception_renderer = options[:exception_renderer] if options[:exception_renderer]
       context.strict_variables = options[:strict_variables] if options[:strict_variables]
       context.strict_filters = options[:strict_filters] if options[:strict_filters]
     end

lib/liquid/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Liquid
-  VERSION = "4.0.0.rc3"
+  VERSION = "4.0.0"
 end

liquid.gemspec

@@ -15,6 +15,7 @@ Gem::Specification.new do |s|
   s.license     = "MIT"
   # s.description = "A secure, non-evaling end user template engine with aesthetic markup."
 
+  s.required_ruby_version     = ">= 2.1.0"
   s.required_rubygems_version = ">= 1.3.7"
 
   s.test_files  = Dir.glob("{test}/**/*")
@@ -24,6 +25,6 @@ Gem::Specification.new do |s|
 
   s.require_path = "lib"
 
-  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rake', '~> 11.3'
   s.add_development_dependency 'minitest'
 end

performance/benchmark.rb

@@ -5,7 +5,7 @@ Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
 profiler = ThemeRunner.new
 
 Benchmark.ips do |x|
-  x.time = 60
+  x.time = 10
   x.warmup = 5
 
   puts
@@ -13,5 +13,6 @@ Benchmark.ips do |x|
   puts
 
   x.report("parse:") { profiler.compile }
-  x.report("parse & run:") { profiler.run }
+  x.report("render:") { profiler.render }
+  x.report("parse & render:") { profiler.run }
 end

performance/theme_runner.rb

@@ -21,53 +21,100 @@ class ThemeRunner
     end
   end
 
-  # Load all templates into memory, do this now so that
-  # we don't profile IO.
+  # Initialize a new liquid ThemeRunner instance
+  # Will load all templates into memory, do this now so that we don't profile IO.
   def initialize
     @tests = Dir[__dir__ + '/tests/**/*.liquid'].collect do |test|
       next if File.basename(test) == 'theme.liquid'
 
       theme_path = File.dirname(test) + '/theme.liquid'
-
-      [File.read(test), (File.file?(theme_path) ? File.read(theme_path) : nil), test]
+      {
+        liquid: File.read(test),
+        layout: (File.file?(theme_path) ? File.read(theme_path) : nil),
+        template_name: test
+      }
     end.compact
+
+    compile_all_tests
   end
 
+  # `compile` will test just the compilation portion of liquid without any templates
   def compile
-    # Dup assigns because will make some changes to them
-
-    @tests.each do |liquid, layout, template_name|
-      tmpl = Liquid::Template.new
-      tmpl.parse(liquid)
-      tmpl = Liquid::Template.new
-      tmpl.parse(layout)
+    @tests.each do |test_hash|
+      Liquid::Template.new.parse(test_hash[:liquid])
+      Liquid::Template.new.parse(test_hash[:layout])
     end
   end
 
+  # `run` is called to benchmark rendering and compiling at the same time
   def run
-    # Dup assigns because will make some changes to them
-    assigns = Database.tables.dup
-
-    @tests.each do |liquid, layout, template_name|
-      # Compute page_tempalte outside of profiler run, uninteresting to profiler
-      page_template = File.basename(template_name, File.extname(template_name))
+    each_test do |liquid, layout, assigns, page_template, template_name|
       compile_and_render(liquid, layout, assigns, page_template, template_name)
     end
   end
 
+  # `render` is called to benchmark just the render portion of liquid
+  def render
+    @compiled_tests.each do |test|
+      tmpl = test[:tmpl]
+      assigns = test[:assigns]
+      layout = test[:layout]
+
+      if layout
+        assigns['content_for_layout'] = tmpl.render!(assigns)
+        layout.render!(assigns)
+      else
+        tmpl.render!(assigns)
+      end
+    end
+  end
+
+  private
+
   def compile_and_render(template, layout, assigns, page_template, template_file)
+    compiled_test = compile_test(template, layout, assigns, page_template, template_file)
+    assigns['content_for_layout'] = compiled_test[:tmpl].render!(assigns)
+    compiled_test[:layout].render!(assigns) if layout
+  end
+
+  def compile_all_tests
+    @compiled_tests = []
+    each_test do |liquid, layout, assigns, page_template, template_name|
+      @compiled_tests << compile_test(liquid, layout, assigns, page_template, template_name)
+    end
+    @compiled_tests
+  end
+
+  def compile_test(template, layout, assigns, page_template, template_file)
+    tmpl = init_template(page_template, template_file)
+    parsed_template = tmpl.parse(template).dup
+
+    if layout
+      parsed_layout = tmpl.parse(layout)
+      { tmpl: parsed_template, assigns: assigns, layout: parsed_layout }
+    else
+      { tmpl: parsed_template, assigns: assigns }
+    end
+  end
+
+  # utility method with similar functionality needed in `compile_all_tests` and `run`
+  def each_test
+    # Dup assigns because will make some changes to them
+    assigns = Database.tables.dup
+
+    @tests.each do |test_hash|
+      # Compute page_template outside of profiler run, uninteresting to profiler
+      page_template = File.basename(test_hash[:template_name], File.extname(test_hash[:template_name]))
+      yield(test_hash[:liquid], test_hash[:layout], assigns, page_template, test_hash[:template_name])
+    end
+  end
+
+  # set up a new Liquid::Template object for use in `compile_and_render` and `compile_test`
+  def init_template(page_template, template_file)
     tmpl = Liquid::Template.new
     tmpl.assigns['page_title'] = 'Page title'
     tmpl.assigns['template'] = page_template
     tmpl.registers[:file_system] = ThemeRunner::FileSystem.new(File.dirname(template_file))
-
-    content_for_layout = tmpl.parse(template).render!(assigns)
-
-    if layout
-      assigns['content_for_layout'] = content_for_layout
-      tmpl.parse(layout).render!(assigns)
-    else
-      content_for_layout
-    end
+    tmpl
   end
 end

test/integration/error_handling_test.rb

@@ -202,22 +202,40 @@ class ErrorHandlingTest < Minitest::Test
     end
   end
 
-  def test_exception_handler_with_string_result
-    template = Liquid::Template.parse('This is an argument error: {{ errors.argument_error }}')
-    output = template.render({ 'errors' => ErrorDrop.new }, exception_handler: ->(e) { '' })
-    assert_equal 'This is an argument error: ', output
-    assert_equal [ArgumentError], template.errors.map(&:class)
-  end
-
-  class InternalError < Liquid::Error
-  end
-
-  def test_exception_handler_with_exception_result
+  def test_default_exception_renderer_with_internal_error
     template = Liquid::Template.parse('This is a runtime error: {{ errors.runtime_error }}', line_numbers: true)
-    handler = ->(e) { e.is_a?(Liquid::Error) ? e : InternalError.new('internal') }
-    output = template.render({ 'errors' => ErrorDrop.new }, exception_handler: handler)
+
+    output = template.render({ 'errors' => ErrorDrop.new })
+
     assert_equal 'This is a runtime error: Liquid error (line 1): internal', output
-    assert_equal [InternalError], template.errors.map(&:class)
+    assert_equal [Liquid::InternalError], template.errors.map(&:class)
+  end
+
+  def test_setting_default_exception_renderer
+    old_exception_renderer = Liquid::Template.default_exception_renderer
+    exceptions = []
+    Liquid::Template.default_exception_renderer = ->(e) { exceptions << e; '' }
+    template = Liquid::Template.parse('This is a runtime error: {{ errors.argument_error }}')
+
+    output = template.render({ 'errors' => ErrorDrop.new })
+
+    assert_equal 'This is a runtime error: ', output
+    assert_equal [Liquid::ArgumentError], template.errors.map(&:class)
+  ensure
+    Liquid::Template.default_exception_renderer = old_exception_renderer if old_exception_renderer
+  end
+
+  def test_exception_renderer_exposing_non_liquid_error
+    template = Liquid::Template.parse('This is a runtime error: {{ errors.runtime_error }}', line_numbers: true)
+    exceptions = []
+    handler = ->(e) { exceptions << e; e.cause }
+
+    output = template.render({ 'errors' => ErrorDrop.new }, exception_renderer: handler)
+
+    assert_equal 'This is a runtime error: runtime error', output
+    assert_equal [Liquid::InternalError], exceptions.map(&:class)
+    assert_equal exceptions, template.errors
+    assert_equal '#<RuntimeError: runtime error>', exceptions.first.cause.inspect
   end
 
   class TestFileSystem

test/integration/parsing_quirks_test.rb

@@ -115,4 +115,8 @@ class ParsingQuirksTest < Minitest::Test
       assert_template_result('12345', "{% for i in (1...5) %}{{ i }}{% endfor %}")
     end
   end
+
+  def test_contains_in_id
+    assert_template_result(' YES ', '{% if containsallshipments == true %} YES {% endif %}', 'containsallshipments' => true)
+  end
 end # ParsingQuirksTest

test/integration/security_test.rb

@@ -63,4 +63,18 @@ class SecurityTest < Minitest::Test
 
     assert_equal [], (Symbol.all_symbols - current_symbols)
   end
+
+  def test_max_depth_nested_blocks_does_not_raise_exception
+    depth = Liquid::Block::MAX_DEPTH
+    code = "{% if true %}" * depth + "rendered" + "{% endif %}" * depth
+    assert_equal "rendered", Template.parse(code).render!
+  end
+
+  def test_more_than_max_depth_nested_blocks_raises_exception
+    depth = Liquid::Block::MAX_DEPTH + 1
+    code = "{% if true %}" * depth + "rendered" + "{% endif %}" * depth
+    assert_raises(Liquid::StackLevelError) do
+      Template.parse(code).render!
+    end
+  end
 end # SecurityTest

test/integration/standard_filter_test.rb

@@ -115,21 +115,29 @@ class StandardFiltersTest < Minitest::Test
     assert_equal '...', @filters.truncate('1234567890', 0)
     assert_equal '1234567890', @filters.truncate('1234567890')
     assert_equal "测试...", @filters.truncate("测试测试测试测试", 5)
+    assert_equal '12341', @filters.truncate("1234567890", 5, 1)
   end
 
   def test_split
     assert_equal ['12', '34'], @filters.split('12~34', '~')
     assert_equal ['A? ', ' ,Z'], @filters.split('A? ~ ~ ~ ,Z', '~ ~ ~')
     assert_equal ['A?Z'], @filters.split('A?Z', '~')
-    # Regexp works although Liquid does not support.
-    assert_equal ['A', 'Z'], @filters.split('AxZ', /x/)
     assert_equal [], @filters.split(nil, ' ')
+    assert_equal ['A', 'Z'], @filters.split('A1Z', 1)
   end
 
   def test_escape
     assert_equal '&lt;strong&gt;', @filters.escape('<strong>')
-    assert_equal nil, @filters.escape(nil)
+    assert_equal '1', @filters.escape(1)
+    assert_equal '2001-02-03', @filters.escape(Date.new(2001, 2, 3))
+    assert_nil @filters.escape(nil)
+  end
+
+  def test_h
     assert_equal '&lt;strong&gt;', @filters.h('<strong>')
+    assert_equal '1', @filters.h(1)
+    assert_equal '2001-02-03', @filters.h(Date.new(2001, 2, 3))
+    assert_nil @filters.h(nil)
   end
 
   def test_escape_once
@@ -138,14 +146,18 @@ class StandardFiltersTest < Minitest::Test
 
   def test_url_encode
     assert_equal 'foo%2B1%40example.com', @filters.url_encode('foo+1@example.com')
-    assert_equal nil, @filters.url_encode(nil)
+    assert_equal '1', @filters.url_encode(1)
+    assert_equal '2001-02-03', @filters.url_encode(Date.new(2001, 2, 3))
+    assert_nil @filters.url_encode(nil)
   end
 
   def test_url_decode
     assert_equal 'foo bar', @filters.url_decode('foo+bar')
     assert_equal 'foo bar', @filters.url_decode('foo%20bar')
     assert_equal 'foo+1@example.com', @filters.url_decode('foo%2B1%40example.com')
-    assert_equal nil, @filters.url_decode(nil)
+    assert_equal '1', @filters.url_decode(1)
+    assert_equal '2001-02-03', @filters.url_decode(Date.new(2001, 2, 3))
+    assert_nil @filters.url_decode(nil)
   end
 
   def test_truncatewords
@@ -154,6 +166,7 @@ class StandardFiltersTest < Minitest::Test
     assert_equal 'one two three', @filters.truncatewords('one two three')
     assert_equal 'Two small (13&#8221; x 5.5&#8221; x 10&#8221; high) baskets fit inside one large basket (13&#8221;...', @filters.truncatewords('Two small (13&#8221; x 5.5&#8221; x 10&#8221; high) baskets fit inside one large basket (13&#8221; x 16&#8221; x 10.5&#8221; high) with cover.', 15)
     assert_equal "测试测试测试测试", @filters.truncatewords('测试测试测试测试', 5)
+    assert_equal 'one two1', @filters.truncatewords("one two three", 2, 1)
   end
 
   def test_strip_html
@@ -169,6 +182,7 @@ class StandardFiltersTest < Minitest::Test
   def test_join
     assert_equal '1 2 3 4', @filters.join([1, 2, 3, 4])
     assert_equal '1 - 2 - 3 - 4', @filters.join([1, 2, 3, 4], ' - ')
+    assert_equal '1121314', @filters.join([1, 2, 3, 4], 1)
   end
 
   def test_sort
@@ -176,6 +190,24 @@ class StandardFiltersTest < Minitest::Test
     assert_equal [{ "a" => 1 }, { "a" => 2 }, { "a" => 3 }, { "a" => 4 }], @filters.sort([{ "a" => 4 }, { "a" => 3 }, { "a" => 1 }, { "a" => 2 }], "a")
   end
 
+  def test_sort_when_property_is_sometimes_missing_puts_nils_last
+    input = [
+      { "price" => 4, "handle" => "alpha" },
+      { "handle" => "beta" },
+      { "price" => 1, "handle" => "gamma" },
+      { "handle" => "delta" },
+      { "price" => 2, "handle" => "epsilon" }
+    ]
+    expectation = [
+      { "price" => 1, "handle" => "gamma" },
+      { "price" => 2, "handle" => "epsilon" },
+      { "price" => 4, "handle" => "alpha" },
+      { "handle" => "delta" },
+      { "handle" => "beta" }
+    ]
+    assert_equal expectation, @filters.sort(input, "price")
+  end
+
   def test_sort_empty_array
     assert_equal [], @filters.sort([], "a")
   end
@@ -310,7 +342,7 @@ class StandardFiltersTest < Minitest::Test
     assert_equal "#{Date.today.year}", @filters.date('today', '%Y')
     assert_equal "#{Date.today.year}", @filters.date('Today', '%Y')
 
-    assert_equal nil, @filters.date(nil, "%B")
+    assert_nil @filters.date(nil, "%B")
 
     assert_equal '', @filters.date('', "%B")
 
@@ -323,15 +355,17 @@ class StandardFiltersTest < Minitest::Test
   def test_first_last
     assert_equal 1, @filters.first([1, 2, 3])
     assert_equal 3, @filters.last([1, 2, 3])
-    assert_equal nil, @filters.first([])
-    assert_equal nil, @filters.last([])
+    assert_nil @filters.first([])
+    assert_nil @filters.last([])
   end
 
   def test_replace
     assert_equal '2 2 2 2', @filters.replace('1 1 1 1', '1', 2)
     assert_equal '2 2 2 2', @filters.replace('1 1 1 1', 1, 2)
+    assert_equal "\\& \\& \\& \\&", @filters.replace('1 1 1 1', '1', "\\&")
     assert_equal '2 1 1 1', @filters.replace_first('1 1 1 1', '1', 2)
     assert_equal '2 1 1 1', @filters.replace_first('1 1 1 1', 1, 2)
+    assert_equal '\\& 1 1 1', @filters.replace_first("1 1 1 1", '1', "\\&")
     assert_template_result '2 1 1 1', "{{ '1 1 1 1' | replace_first: '1', 2 }}"
   end
 

test/integration/tags/include_tag_test.rb

@@ -137,7 +137,7 @@ class IncludeTagTest < Minitest::Test
 
     Liquid::Template.file_system = infinite_file_system.new
 
-    assert_raises(Liquid::StackLevelError, SystemStackError) do
+    assert_raises(Liquid::StackLevelError) do
       Template.parse("{% include 'loop' %}").render!
     end
   end
@@ -235,4 +235,11 @@ class IncludeTagTest < Minitest::Test
 
     assert_template_result "Product: Draft 151cm ", "{% assign page = 'product' %}{% include page for foo %}", "foo" => { 'title' => 'Draft 151cm' }
   end
+
+  def test_including_with_strict_variables
+    template = Liquid::Template.parse("{% include 'simple' %}", error_mode: :warn)
+    template.render(nil, strict_variables: true)
+
+    assert_equal [], template.errors
+  end
 end # IncludeTagTest

test/integration/template_test.rb

@@ -215,16 +215,20 @@ class TemplateTest < Minitest::Test
     assert_equal 'ruby error in drop', e.message
   end
 
-  def test_exception_handler_doesnt_reraise_if_it_returns_false
+  def test_exception_renderer_that_returns_string
     exception = nil
-    Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_handler: ->(e) { exception = e; false })
+    handler = ->(e) { exception = e; '<!-- error -->' }
+
+    output = Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_renderer: handler)
+
     assert exception.is_a?(Liquid::ZeroDivisionError)
+    assert_equal '<!-- error -->', output
   end
 
-  def test_exception_handler_does_reraise_if_it_returns_true
+  def test_exception_renderer_that_raises
     exception = nil
     assert_raises(Liquid::ZeroDivisionError) do
-      Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_handler: ->(e) { exception = e; true })
+      Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_renderer: ->(e) { exception = e; raise })
     end
     assert exception.is_a?(Liquid::ZeroDivisionError)
   end
@@ -257,6 +261,15 @@ class TemplateTest < Minitest::Test
     assert_equal 'Liquid error: undefined variable d', t.errors[2].message
   end
 
+  def test_nil_value_does_not_raise
+    Liquid::Template.error_mode = :strict
+    t = Template.parse("some{{x}}thing")
+    result = t.render!({ 'x' => nil }, strict_variables: true)
+
+    assert_equal 0, t.errors.count
+    assert_equal 'something', result
+  end
+
   def test_undefined_variables_raise
     t = Template.parse("{{x}} {{y}} {{z.a}} {{z.b}} {{z.c.d}}")
 

test/integration/variable_test.rb

@@ -89,4 +89,8 @@ class VariableTest < Minitest::Test
   def test_multiline_variable
     assert_equal 'worked', Template.parse("{{\ntest\n}}").render!('test' => 'worked')
   end
+
+  def test_render_symbol
+    assert_template_result 'bar', '{{ foo }}', 'foo' => :bar
+  end
 end

test/test_helper.rb

@@ -2,7 +2,6 @@
 
 ENV["MT_NO_EXPECTATIONS"] = "1"
 require 'minitest/autorun'
-require 'spy/integration'
 
 $LOAD_PATH.unshift(File.join(File.expand_path(__dir__), '..', 'lib'))
 require 'liquid.rb'

test/unit/condition_unit_test.rb

@@ -64,6 +64,14 @@ class ConditionUnitTest < Minitest::Test
     assert_evaluates_argument_error '1', '<=', 0
   end
 
+  def test_hash_compare_backwards_compatibility
+    assert_nil Condition.new({}, '>', 2).evaluate
+    assert_nil Condition.new(2, '>', {}).evaluate
+    assert_equal false, Condition.new({}, '==', 2).evaluate
+    assert_equal true, Condition.new({ 'a' => 1 }, '==', { 'a' => 1 }).evaluate
+    assert_equal true, Condition.new({ 'a' => 2 }, 'contains', 'a').evaluate
+  end
+
   def test_contains_works_on_arrays
     @context = Liquid::Context.new
     @context['array'] = [1, 2, 3, 4, 5]

test/unit/context_unit_test.rb

@@ -70,10 +70,6 @@ class ContextUnitTest < Minitest::Test
     @context = Liquid::Context.new
   end
 
-  def teardown
-    Spy.teardown
-  end
-
   def test_variables
     @context['string'] = 'string'
     assert_equal 'string', @context['string']
@@ -98,12 +94,12 @@ class ContextUnitTest < Minitest::Test
     assert_equal false, @context['bool']
 
     @context['nil'] = nil
-    assert_equal nil, @context['nil']
-    assert_equal nil, @context['nil']
+    assert_nil @context['nil']
+    assert_nil @context['nil']
   end
 
   def test_variables_not_existing
-    assert_equal nil, @context['does_not_exist']
+    assert_nil @context['does_not_exist']
   end
 
   def test_scoping
@@ -185,7 +181,7 @@ class ContextUnitTest < Minitest::Test
     @context['test'] = 'test'
     assert_equal 'test', @context['test']
     @context.pop
-    assert_equal nil, @context['test']
+    assert_nil @context['test']
   end
 
   def test_hierachical_data
@@ -300,7 +296,7 @@ class ContextUnitTest < Minitest::Test
     @context['hash'] = { 'first' => 'Hello' }
 
     assert_equal 1, @context['array.first']
-    assert_equal nil, @context['array["first"]']
+    assert_nil @context['array["first"]']
     assert_equal 'Hello', @context['hash["first"]']
   end
 
@@ -450,14 +446,10 @@ class ContextUnitTest < Minitest::Test
     assert_equal @context, @context['category'].context
   end
 
-  def test_use_empty_instead_of_any_in_interrupt_handling_to_avoid_lots_of_unnecessary_object_allocations
-    mock_any = Spy.on_instance_method(Array, :any?)
-    mock_empty = Spy.on_instance_method(Array, :empty?)
-
-    @context.interrupt?
-
-    refute mock_any.has_been_called?
-    assert mock_empty.has_been_called?
+  def test_interrupt_avoids_object_allocations
+    assert_no_object_allocations do
+      @context.interrupt?
+    end
   end
 
   def test_context_initialization_with_a_proc_in_environment
@@ -480,4 +472,18 @@ class ContextUnitTest < Minitest::Test
     context = Context.new
     assert_equal 'hi', context.apply_global_filter('hi')
   end
+
+  private
+
+  def assert_no_object_allocations
+    unless RUBY_ENGINE == 'ruby'
+      skip "stackprof needed to count object allocations"
+    end
+    require 'stackprof'
+
+    profile = StackProf.run(mode: :object) do
+      yield
+    end
+    assert_equal 0, profile[:samples]
+  end
 end # ContextTest

test/unit/lexer_unit_test.rb

@@ -19,7 +19,7 @@ class LexerUnitTest < Minitest::Test
   end
 
   def test_comparison
-    tokens = Lexer.new('== <> contains').tokenize
+    tokens = Lexer.new('== <> contains ').tokenize
     assert_equal [[:comparison, '=='], [:comparison, '<>'], [:comparison, 'contains'], [:end_of_string]], tokens
   end
 

test/unit/strainer_unit_test.rb

@@ -133,4 +133,32 @@ class StrainerUnitTest < Minitest::Test
     strainer.class.add_filter(PublicMethodOverrideFilter)
     assert strainer.class.filter_methods.include?('public_filter')
   end
+
+  module LateAddedFilter
+    def late_added_filter(input)
+      "filtered"
+    end
+  end
+
+  def test_global_filter_clears_cache
+    assert_equal 'input', Strainer.create(nil).invoke('late_added_filter', 'input')
+    Strainer.global_filter(LateAddedFilter)
+    assert_equal 'filtered', Strainer.create(nil).invoke('late_added_filter', 'input')
+  end
+
+  def test_add_filter_does_not_include_already_included_module
+    mod = Module.new do
+      class << self
+        attr_accessor :include_count
+        def included(mod)
+          self.include_count += 1
+        end
+      end
+      self.include_count = 0
+    end
+    strainer = Context.new.strainer
+    strainer.class.add_filter(mod)
+    strainer.class.add_filter(mod)
+    assert_equal 1, mod.include_count
+  end
 end # StrainerTest