Optimize range iteration

For loops support a range syntax for iterating a fixed number of times,
which looks like this.

```liquid
{% for i in range (1..100) %}
        ...
{% endfor %}
```

Previously, we converted these ranges to arrays using `to_a`, which
initialized an array containing each number in the range. Since all we
use these ranges for is iteration, this is far less efficient than
using a range iterator.

Doing this means that iterating over ranges now takes O(1) rather than O(n)
memory. See the PR for more benchmarks.

* Remove to_a cast on ranges
* Add ReversableRange iterator
* Add custom range-specific slicing logic

.gitignore

@@ -7,4 +7,3 @@ pkg
 .ruby-version
 Gemfile.lock
 .bundle
-.byebug_history

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2019-04-22 19:11:24 -0400 using RuboCop version 0.53.0.
+# on 2019-03-19 11:04:37 -0400 using RuboCop version 0.53.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -46,18 +46,18 @@ Lint/Void:
   Exclude:
     - 'lib/liquid/parse_context.rb'
 
-# Offense count: 53
+# Offense count: 54
 Metrics/AbcSize:
   Max: 56
 
 # Offense count: 12
 Metrics/CyclomaticComplexity:
-  Max: 13
+  Max: 12
 
 # Offense count: 112
 # Configuration parameters: CountComments.
 Metrics/MethodLength:
-  Max: 38
+  Max: 37
 
 # Offense count: 8
 Metrics/PerceivedComplexity:
@@ -90,7 +90,7 @@ Naming/UncommunicativeMethodParamName:
     - 'test/integration/template_test.rb'
     - 'test/unit/condition_unit_test.rb'
 
-# Offense count: 12
+# Offense count: 10
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: prefer_alias, prefer_alias_method
@@ -253,7 +253,7 @@ Style/WhileUntilModifier:
   Exclude:
     - 'lib/liquid/tags/case.rb'
 
-# Offense count: 648
+# Offense count: 640
 # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
 # URISchemes: http, https
 Metrics/LineLength:

.travis.yml

@@ -6,21 +6,18 @@ rvm:
   - 2.3
   - 2.4
   - 2.5
-  - &latest_ruby 2.6
   - ruby-head
   - jruby-head
 #  - rbx-2
 
+sudo: false
+
 addons:
   apt:
     packages:
     - libgmp3-dev
 
 matrix:
-  include:
-    - rvm: *latest_ruby
-      script: bundle exec rake memory_profile:run
-      name: Profiling Memory Usage
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head

Gemfile

@@ -8,7 +8,6 @@ gemspec
 group :benchmark, :test do
   gem 'benchmark-ips'
   gem 'memory_profiler'
-  gem 'terminal-table'
 
   install_if -> { RUBY_PLATFORM !~ /mingw|mswin|java/ } do
     gem 'stackprof'
@@ -19,6 +18,6 @@ group :test do
   gem 'rubocop', '~> 0.53.0'
 
   platform :mri do
-    gem 'liquid-c', github: 'Shopify/liquid-c', ref: 'liquid-tag'
+    gem 'liquid-c', github: 'Shopify/liquid-c', ref: 'reversable-range'
   end
 end

lib/liquid.rb

@@ -74,6 +74,7 @@ require 'liquid/condition'
 require 'liquid/utils'
 require 'liquid/tokenizer'
 require 'liquid/parse_context'
+require 'liquid/reversable_range'
 
 # Load all the tags of the standard library
 #

lib/liquid/block.rb

@@ -13,7 +13,6 @@ module Liquid
       end
     end
 
-    # For backwards compatibility
     def render(context)
       @body.render(context)
     end

lib/liquid/block_body.rb

@@ -1,7 +1,6 @@
 module Liquid
   class BlockBody
-    LiquidTagToken = /\A\s*(\w+)\s*(.*?)\z/o
+    FullToken = /\A#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
-    FullToken = /\A#{TagStart}#{WhitespaceControl}?(\s*)(\w+)(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
     ContentOfVariable = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
     WhitespaceOrNothing = /\A\s*\z/
     TAGSTART = "{%".freeze
@@ -14,42 +13,8 @@ module Liquid
       @blank = true
     end
 
-    def parse(tokenizer, parse_context, &block)
+    def parse(tokenizer, parse_context)
       parse_context.line_number = tokenizer.line_number
-
-      if tokenizer.for_liquid_tag
-        parse_for_liquid_tag(tokenizer, parse_context, &block)
-      else
-        parse_for_document(tokenizer, parse_context, &block)
-      end
-    end
-
-    private def parse_for_liquid_tag(tokenizer, parse_context)
-      while token = tokenizer.shift
-        unless token.empty? || token =~ WhitespaceOrNothing
-          unless token =~ LiquidTagToken
-            # line isn't empty but didn't match tag syntax, yield and let the
-            # caller raise a syntax error
-            return yield token, token
-          end
-          tag_name = $1
-          markup = $2
-          unless tag = registered_tags[tag_name]
-            # end parsing if we reach an unknown tag and let the caller decide
-            # determine how to proceed
-            return yield tag_name, markup
-          end
-          new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
-          @blank &&= new_tag.blank?
-          @nodelist << new_tag
-        end
-        parse_context.line_number = tokenizer.line_number
-      end
-
-      yield nil, nil
-    end
-
-    private def parse_for_document(tokenizer, parse_context, &block)
       while token = tokenizer.shift
         next if token.empty?
         case
@@ -58,20 +23,9 @@ module Liquid
           unless token =~ FullToken
             raise_missing_tag_terminator(token, parse_context)
           end
-          tag_name = $2
+          tag_name = $1
-          markup = $4
+          markup = $2
-
+          # fetch the tag from registered blocks
-          if parse_context.line_number
-            # newlines inside the tag should increase the line number,
-            # particularly important for multiline {% liquid %} tags
-            parse_context.line_number += $1.count("\n".freeze) + $3.count("\n".freeze)
-          end
-
-          if tag_name == 'liquid'.freeze
-            liquid_tag_tokenizer = Tokenizer.new(markup, line_number: parse_context.line_number, for_liquid_tag: true)
-            next parse_for_liquid_tag(liquid_tag_tokenizer, parse_context, &block)
-          end
-
           unless tag = registered_tags[tag_name]
             # end parsing if we reach an unknown tag and let the caller decide
             # determine how to proceed
@@ -113,23 +67,19 @@ module Liquid
     end
 
     def render(context)
-      render_to_output_buffer(context, '')
+      output = []
-    end
-
-    def render_to_output_buffer(context, output)
       context.resource_limits.render_score += @nodelist.length
 
       idx = 0
       while node = @nodelist[idx]
-        previous_output_size = output.bytesize
-
         case node
         when String
+          check_resources(context, node)
           output << node
         when Variable
-          render_node(context, output, node)
+          render_node_to_output(node, output, context)
         when Block
-          render_node(context, node.blank? ? '' : output, node)
+          render_node_to_output(node, output, context, node.blank?)
           break if context.interrupt? # might have happened in a for-block
         when Continue, Break
           # If we get an Interrupt that means the block must stop processing. An
@@ -138,30 +88,34 @@ module Liquid
           context.push_interrupt(node.interrupt)
           break
         else # Other non-Block tags
-          render_node(context, output, node)
+          render_node_to_output(node, output, context)
           break if context.interrupt? # might have happened through an include
         end
         idx += 1
-
-        raise_if_resource_limits_reached(context, output.bytesize - previous_output_size)
       end
 
-      output
+      output.join
     end
 
     private
 
-    def render_node(context, output, node)
+    def render_node_to_output(node, output, context, skip_output = false)
-      node.render_to_output_buffer(context, output)
+      node_output = node.render(context)
+      node_output = node_output.is_a?(Array) ? node_output.join : node_output.to_s
+      check_resources(context, node_output)
+      output << node_output unless skip_output
+    rescue MemoryError => e
+      raise e
     rescue UndefinedVariable, UndefinedDropMethod, UndefinedFilter => e
       context.handle_error(e, node.line_number)
+      output << nil
     rescue ::StandardError => e
       line_number = node.is_a?(String) ? nil : node.line_number
       output << context.handle_error(e, line_number)
     end
 
-    def raise_if_resource_limits_reached(context, length)
+    def check_resources(context, node_output)
-      context.resource_limits.render_length += length
+      context.resource_limits.render_length += node_output.bytesize
       return unless context.resource_limits.reached?
       raise MemoryError.new("Memory limits exceeded".freeze)
     end

lib/liquid/context.rb

@@ -20,17 +20,21 @@ module Liquid
       @scopes           = [(outer_scope || {})]
       @registers        = registers
       @errors           = []
-      @interrupts       = []
-      @filters          = []
-      @global_filter    = nil
       @partial          = false
       @strict_variables = false
       @resource_limits  = resource_limits || ResourceLimits.new(Template.default_resource_limits)
+      squash_instance_assigns_with_environments
+
+      @this_stack_used = false
 
       self.exception_renderer = Template.default_exception_renderer
-      self.exception_renderer = ->(e) { raise } if rethrow_errors
+      if rethrow_errors
+        self.exception_renderer = ->(e) { raise }
+      end
 
-      squash_instance_assigns_with_environments
+      @interrupts = []
+      @filters = []
+      @global_filter = nil
     end
 
     def warnings
@@ -83,9 +87,9 @@ module Liquid
     end
 
     # Push new local scope on the stack. use <tt>Context#stack</tt> instead
-    def push
+    def push(new_scope = {})
-      @scopes.unshift({})
+      @scopes.unshift(new_scope)
-      raise StackLevelError, "Nesting too deep".freeze if @scopes.length > (Block::MAX_DEPTH + 1)
+      raise StackLevelError, "Nesting too deep".freeze if @scopes.length > Block::MAX_DEPTH
     end
 
     # Merge a hash of variables in the current local scope
@@ -107,15 +111,31 @@ module Liquid
     #   end
     #
     #   context['var]  #=> nil
-    def stack
+    def stack(new_scope = nil)
-      push
+      old_stack_used = @this_stack_used
+      if new_scope
+        push(new_scope)
+        @this_stack_used = true
+      else
+        @this_stack_used = false
+      end
+
       yield
     ensure
-      pop
+      pop if @this_stack_used
+      @this_stack_used = old_stack_used
+    end
+
+    def clear_instance_assigns
+      @scopes[0] = {}
     end
 
     # Only allow String, Numeric, Hash, Array, Proc, Boolean or <tt>Liquid::Drop</tt>
     def []=(key, value)
+      unless @this_stack_used
+        @this_stack_used = true
+        push({})
+      end
       @scopes[0][key] = value
     end
 
@@ -143,12 +163,27 @@ module Liquid
     def find_variable(key, raise_on_not_found: true)
       # This was changed from find() to find_index() because this is a very hot
       # path and find_index() is optimized in MRI to reduce object allocation
+      index = @scopes.find_index { |s| s.key?(key) }
+      scope = @scopes[index] if index
 
-      scope = (index = @scopes.find_index { |s| s.key?(key)  }) && @scopes[index]
+      variable = nil
-      scope ||= (index = @environments.find_index { |s| s.key?(key) }) && @environments[index]
-      scope ||= {}
 
-      variable = lookup_and_evaluate(scope, key, raise_on_not_found: raise_on_not_found).to_liquid
+      if scope.nil?
+        @environments.each do |e|
+          variable = lookup_and_evaluate(e, key, raise_on_not_found: raise_on_not_found)
+          # When lookup returned a value OR there is no value but the lookup also did not raise
+          # then it is the value we are looking for.
+          if !variable.nil? || @strict_variables && raise_on_not_found
+            scope = e
+            break
+          end
+        end
+      end
+
+      scope ||= @environments.last || @scopes.last
+      variable ||= lookup_and_evaluate(scope, key, raise_on_not_found: raise_on_not_found)
+
+      variable = variable.to_liquid
       variable.context = self if variable.respond_to?(:context=)
 
       variable

lib/liquid/parser.rb

@@ -44,14 +44,11 @@ module Liquid
       tok[0] == type
     end
 
-    SINGLE_TOKEN_EXPRESSION_TYPES = [:string, :number].freeze
-    private_constant :SINGLE_TOKEN_EXPRESSION_TYPES
-
     def expression
       token = @tokens[@p]
       if token[0] == :id
         variable_signature
-      elsif SINGLE_TOKEN_EXPRESSION_TYPES.include? token[0]
+      elsif [:string, :number].include? token[0]
         consume
       elsif token.first == :open_round
         consume

lib/liquid/profiler/hooks.rb

@@ -1,23 +1,23 @@
 module Liquid
   class BlockBody
-    def render_node_with_profiling(context, output, node)
+    def render_node_with_profiling(node, output, context, skip_output = false)
       Profiler.profile_node_render(node) do
-        render_node_without_profiling(context, output, node)
+        render_node_without_profiling(node, output, context, skip_output)
       end
     end
 
-    alias_method :render_node_without_profiling, :render_node
+    alias_method :render_node_without_profiling, :render_node_to_output
-    alias_method :render_node, :render_node_with_profiling
+    alias_method :render_node_to_output, :render_node_with_profiling
   end
 
   class Include < Tag
-    def render_to_output_buffer_with_profiling(context, output)
+    def render_with_profiling(context)
       Profiler.profile_children(context.evaluate(@template_name_expr).to_s) do
-        render_to_output_buffer_without_profiling(context, output)
+        render_without_profiling(context)
       end
     end
 
-    alias_method :render_to_output_buffer_without_profiling, :render_to_output_buffer
+    alias_method :render_without_profiling, :render
-    alias_method :render_to_output_buffer, :render_to_output_buffer_with_profiling
+    alias_method :render, :render_with_profiling
   end
 end

lib/liquid/range_lookup.rb

@@ -6,7 +6,7 @@ module Liquid
       if start_obj.respond_to?(:evaluate) || end_obj.respond_to?(:evaluate)
         new(start_obj, end_obj)
       else
-        start_obj.to_i..end_obj.to_i
+        ReversableRange.new(start_obj.to_i, end_obj.to_i)
       end
     end
 
@@ -18,7 +18,7 @@ module Liquid
     def evaluate(context)
       start_int = to_integer(context.evaluate(@start_obj))
       end_int = to_integer(context.evaluate(@end_obj))
-      start_int..end_int
+      ReversableRange.new(start_int, end_int)
     end
 
     private

lib/liquid/reversable_range.rb

@@ -0,0 +1,77 @@
+module Liquid
+  class ReversableRange
+    include Enumerable
+
+    def initialize(min, max)
+      @min = min
+      @max = max
+      @reversed = false
+    end
+
+    def each
+      if reversed
+        index = max
+        while index >= min
+          yield index
+          index -= 1
+        end
+      else
+        index = min
+        while index <= max
+          yield index
+          index += 1
+        end
+      end
+    end
+
+    def reverse!
+      @reversed = !reversed
+      self
+    end
+
+    def empty?
+      max < min
+    end
+
+    def size
+      difference = max - min
+      if difference > 0
+        difference + 1
+      else
+        0
+      end
+    end
+
+    def load_slice(from, to = nil)
+      to ||= max
+      slice_max = [max, to].min
+      slice_min = [min, from].max
+      range = ReversableRange.new(slice_min, slice_max)
+      range.reverse! if reversed
+      range
+    end
+
+    def ==(other)
+      other.is_a?(self.class) &&
+        other.min == min &&
+        other.max == max &&
+        other.reversed == reversed
+    end
+
+    def to_s
+      if reversed
+        "#{max}..#{min}"
+      else
+        "#{min}..#{max}"
+      end
+    end
+
+    def to_liquid
+      self
+    end
+
+    protected
+
+    attr_reader :min, :max, :reversed
+  end
+end

lib/liquid/standardfilters.rb

@@ -79,7 +79,7 @@ module Liquid
       truncate_string_str = truncate_string.to_s
       l = length - truncate_string_str.length
       l = 0 if l < 0
-      input_str.length > length ? input_str[0...l].concat(truncate_string_str) : input_str
+      input_str.length > length ? input_str[0...l] + truncate_string_str : input_str
     end
 
     def truncatewords(input, words = 15, truncate_string = "...".freeze)
@@ -88,7 +88,7 @@ module Liquid
       words = Utils.to_integer(words)
       l = words - 1
       l = 0 if l < 0
-      wordlist.length > l ? wordlist[0..l].join(" ".freeze).concat(truncate_string.to_s) : input
+      wordlist.length > l ? wordlist[0..l].join(" ".freeze) + truncate_string.to_s : input
     end
 
     # Split input string into an array of substrings separated by given pattern.

lib/liquid/tag.rb

@@ -5,8 +5,8 @@ module Liquid
     include ParserSwitching
 
     class << self
-      def parse(tag_name, markup, tokenizer, parse_context)
+      def parse(tag_name, markup, tokenizer, options)
-        tag = new(tag_name, markup, parse_context)
+        tag = new(tag_name, markup, options)
         tag.parse(tokenizer)
         tag
       end
@@ -36,14 +36,6 @@ module Liquid
       ''.freeze
     end
 
-    # For backwards compatibility with custom tags. In a future release, the semantics
-    # of the `render_to_output_buffer` method will become the default and the `render`
-    # method will be removed.
-    def render_to_output_buffer(context, output)
-      output << render(context)
-      output
-    end
-
     def blank?
       false
     end

lib/liquid/tags/assign.rb

@@ -10,10 +10,6 @@ module Liquid
   class Assign < Tag
     Syntax = /(#{VariableSignature}+)\s*=\s*(.*)\s*/om
 
-    def self.syntax_error_translation_key
-      "errors.syntax.assign".freeze
-    end
-
     attr_reader :to, :from
 
     def initialize(tag_name, markup, options)
@@ -22,15 +18,15 @@ module Liquid
         @to = $1
         @from = Variable.new($2, options)
       else
-        raise SyntaxError.new(options[:locale].t(self.class.syntax_error_translation_key))
+        raise SyntaxError.new options[:locale].t("errors.syntax.assign".freeze)
       end
     end
 
-    def render_to_output_buffer(context, output)
+    def render(context)
       val = @from.render(context)
       context.scopes.last[@to] = val
       context.resource_limits.assign_score += assign_score_of(val)
-      output
+      ''.freeze
     end
 
     def blank?

lib/liquid/tags/capture.rb

@@ -22,12 +22,11 @@ module Liquid
       end
     end
 
-    def render_to_output_buffer(context, output)
+    def render(context)
-      previous_output_size = output.bytesize
+      output = super
-      super
       context.scopes.last[@to] = output
-      context.resource_limits.assign_score += (output.bytesize - previous_output_size)
+      context.resource_limits.assign_score += output.bytesize
-      output
+      ''.freeze
     end
 
     def blank?

lib/liquid/tags/case.rb

@@ -38,22 +38,22 @@ module Liquid
       end
     end
 
-    def render_to_output_buffer(context, output)
+    def render(context)
       context.stack do
         execute_else_block = true
 
+        output = ''
         @blocks.each do |block|
           if block.else?
-            block.attachment.render_to_output_buffer(context, output) if execute_else_block
+            return block.attachment.render(context) if execute_else_block
           elsif block.evaluate(context)
             execute_else_block = false
-            block.attachment.render_to_output_buffer(context, output)
+            output << block.attachment.render(context)
           end
         end
-      end
-
         output
       end
+    end
 
     private
 

lib/liquid/tags/comment.rb

@@ -1,7 +1,7 @@
 module Liquid
   class Comment < Block
-    def render_to_output_buffer(_context, output)
+    def render(_context)
-      output
+      ''.freeze
     end
 
     def unknown_tag(_tag, _markup, _tokens)

lib/liquid/tags/cycle.rb

@@ -31,29 +31,18 @@ module Liquid
       end
     end
 
-    def render_to_output_buffer(context, output)
+    def render(context)
       context.registers[:cycle] ||= {}
 
       context.stack do
         key = context.evaluate(@name)
         iteration = context.registers[:cycle][key].to_i
-
+        result = context.evaluate(@variables[iteration])
-        val = context.evaluate(@variables[iteration])
-
-        if val.is_a?(Array)
-          val = val.join
-        elsif !val.is_a?(String)
-          val = val.to_s
-        end
-
-        output << val
-
         iteration += 1
         iteration  = 0 if iteration >= @variables.size
         context.registers[:cycle][key] = iteration
+        result
       end
-
-      output
     end
 
     private

lib/liquid/tags/decrement.rb

@@ -23,12 +23,11 @@ module Liquid
       @variable = markup.strip
     end
 
-    def render_to_output_buffer(context, output)
+    def render(context)
       value = context.environments.first[@variable] ||= 0
       value -= 1
       context.environments.first[@variable] = value
-      output << value.to_s
+      value.to_s
-      output
     end
   end
 

lib/liquid/tags/echo.rb

@@ -1,24 +0,0 @@
-module Liquid
-  # Echo outputs an expression
-  #
-  #   {% echo monkey %}
-  #   {% echo user.name %}
-  #
-  # This is identical to variable output syntax, like {{ foo }}, but works
-  # inside {% liquid %} tags. The full syntax is supported, including filters:
-  #
-  #   {% echo user | link %}
-  #
-  class Echo < Tag
-    def initialize(tag_name, markup, parse_context)
-      super
-      @variable = Variable.new(markup, parse_context)
-    end
-
-    def render(context)
-      @variable.render(context)
-    end
-  end
-
-  Template.register_tag('echo'.freeze, Echo)
-end

lib/liquid/tags/for.rb

@@ -70,16 +70,14 @@ module Liquid
       @else_block = BlockBody.new
     end
 
-    def render_to_output_buffer(context, output)
+    def render(context)
       segment = collection_segment(context)
 
       if segment.empty?
-        render_else(context, output)
+        render_else(context)
       else
-        render_segment(context, output, segment)
+        render_segment(context, segment)
       end
-
-      output
     end
 
     protected
@@ -135,7 +133,6 @@ module Liquid
       end
 
       collection = context.evaluate(@collection_name)
-      collection = collection.to_a if collection.is_a?(Range)
 
       limit_value = context.evaluate(@limit)
       to = if limit_value.nil?
@@ -147,14 +144,16 @@ module Liquid
       segment = Utils.slice_collection(collection, from, to)
       segment.reverse! if @reversed
 
-      offsets[@name] = from + segment.length
+      offsets[@name] = from + segment.size
 
       segment
     end
 
-    def render_segment(context, output, segment)
+    def render_segment(context, segment)
       for_stack = context.registers[:for_stack] ||= []
-      length = segment.length
+      length = segment.size
+
+      result = ''
 
       context.stack do
         loop_vars = Liquid::ForloopDrop.new(@name, length, for_stack[-1])
@@ -166,7 +165,7 @@ module Liquid
 
           segment.each do |item|
             context[@variable_name] = item
-            @for_block.render_to_output_buffer(context, output)
+            result << @for_block.render(context)
             loop_vars.send(:increment!)
 
             # Handle any interrupts if they exist.
@@ -181,7 +180,7 @@ module Liquid
         end
       end
 
-      output
+      result
     end
 
     def set_attribute(key, expr)
@@ -197,12 +196,8 @@ module Liquid
       end
     end
 
-    def render_else(context, output)
+    def render_else(context)
-      if @else_block
+      @else_block ? @else_block.render(context) : ''.freeze
-        @else_block.render_to_output_buffer(context, output)
-      else
-        output
-      end
     end
 
     class ParseTreeVisitor < Liquid::ParseTreeVisitor

lib/liquid/tags/if.rb

@@ -39,16 +39,15 @@ module Liquid
       end
     end
 
-    def render_to_output_buffer(context, output)
+    def render(context)
       context.stack do
         @blocks.each do |block|
           if block.evaluate(context)
-            return block.attachment.render_to_output_buffer(context, output)
+            return block.attachment.render(context)
           end
         end
+        ''.freeze
       end
-
-      output
     end
 
     private

lib/liquid/tags/ifchanged.rb

@@ -1,17 +1,16 @@
 module Liquid
   class Ifchanged < Block
-    def render_to_output_buffer(context, output)
+    def render(context)
       context.stack do
-        block_output = ''
+        output = super
-        super(context, block_output)
-
-        if block_output != context.registers[:ifchanged]
-          context.registers[:ifchanged] = block_output
-          output << block_output
-        end
-      end
 
+        if output != context.registers[:ifchanged]
+          context.registers[:ifchanged] = output
           output
+        else
+          ''.freeze
+        end
+      end
     end
   end
 

lib/liquid/tags/include.rb

@@ -42,7 +42,7 @@ module Liquid
     def parse(_tokens)
     end
 
-    def render_to_output_buffer(context, output)
+    def render(context)
       template_name = context.evaluate(@template_name_expr)
       raise ArgumentError.new(options[:locale].t("errors.argument.include")) unless template_name
 
@@ -66,21 +66,19 @@ module Liquid
           end
 
           if variable.is_a?(Array)
-            variable.each do |var|
+            variable.collect do |var|
               context[context_variable_name] = var
-              partial.render_to_output_buffer(context, output)
+              partial.render(context)
             end
           else
             context[context_variable_name] = variable
-            partial.render_to_output_buffer(context, output)
+            partial.render(context)
           end
         end
       ensure
         context.template_name = old_template_name
         context.partial = old_partial
       end
-
-      output
     end
 
     private

lib/liquid/tags/increment.rb

@@ -20,11 +20,10 @@ module Liquid
       @variable = markup.strip
     end
 
-    def render_to_output_buffer(context, output)
+    def render(context)
       value = context.environments.first[@variable] ||= 0
       context.environments.first[@variable] = value + 1
-      output << value.to_s
+      value.to_s
-      output
     end
   end
 

lib/liquid/tags/raw.rb

@@ -22,9 +22,8 @@ module Liquid
       raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_never_closed".freeze, block_name: block_name))
     end
 
-    def render_to_output_buffer(_context, output)
+    def render(_context)
-      output << @body
+      @body
-      output
     end
 
     def nodelist

lib/liquid/tags/table_row.rb

@@ -18,7 +18,7 @@ module Liquid
       end
     end
 
-    def render_to_output_buffer(context, output)
+    def render(context)
       collection = context.evaluate(@collection_name) or return ''.freeze
 
       from = @attributes.key?('offset'.freeze) ? context.evaluate(@attributes['offset'.freeze]).to_i : 0
@@ -30,7 +30,7 @@ module Liquid
 
       cols = context.evaluate(@attributes['cols'.freeze]).to_i
 
-      output << "<tr class=\"row1\">\n"
+      result = "<tr class=\"row1\">\n"
       context.stack do
         tablerowloop = Liquid::TablerowloopDrop.new(length, cols)
         context['tablerowloop'.freeze] = tablerowloop
@@ -38,20 +38,17 @@ module Liquid
         collection.each do |item|
           context[@variable_name] = item
 
-          output << "<td class=\"col#{tablerowloop.col}\">"
+          result << "<td class=\"col#{tablerowloop.col}\">" << super << '</td>'
-          super
-          output << '</td>'
 
           if tablerowloop.col_last && !tablerowloop.last
-            output << "</tr>\n<tr class=\"row#{tablerowloop.row + 1}\">"
+            result << "</tr>\n<tr class=\"row#{tablerowloop.row + 1}\">"
           end
 
           tablerowloop.send(:increment!)
         end
       end
-
+      result << "</tr>\n"
-      output << "</tr>\n"
+      result
-      output
     end
 
     class ParseTreeVisitor < Liquid::ParseTreeVisitor

lib/liquid/tags/unless.rb

@@ -6,23 +6,23 @@ module Liquid
   #   {% unless x < 0 %} x is greater than zero {% endunless %}
   #
   class Unless < If
-    def render_to_output_buffer(context, output)
+    def render(context)
       context.stack do
         # First condition is interpreted backwards ( if not )
         first_block = @blocks.first
         unless first_block.evaluate(context)
-          return first_block.attachment.render_to_output_buffer(context, output)
+          return first_block.attachment.render(context)
         end
 
         # After the first condition unless works just like if
         @blocks[1..-1].each do |block|
           if block.evaluate(context)
-            return block.attachment.render_to_output_buffer(context, output)
+            return block.attachment.render(context)
-          end
           end
         end
 
-      output
+        ''.freeze
+      end
     end
   end
 

lib/liquid/template.rb

@@ -50,7 +50,7 @@ module Liquid
       private
 
       def lookup_class(name)
-        Object.const_get(name)
+        name.split("::").reject(&:empty?).reduce(Object) { |scope, const| scope.const_get(const) }
       end
     end
 
@@ -187,12 +187,9 @@ module Liquid
         raise ArgumentError, "Expected Hash or Liquid::Context as parameter"
       end
 
-      output = nil
-
       case args.last
       when Hash
         options = args.pop
-        output = options[:output] if options[:output]
 
         registers.merge!(options[:registers]) if options[:registers].is_a?(Hash)
 
@@ -207,9 +204,10 @@ module Liquid
       begin
         # render the nodelist.
         # for performance reasons we get an array back here. join will make a string out of it.
-        with_profiling(context) do
+        result = with_profiling(context) do
-          @root.render_to_output_buffer(context, output || '')
+          @root.render(context)
         end
+        result.respond_to?(:join) ? result.join : result
       rescue Liquid::MemoryError => e
         context.handle_error(e)
       ensure
@@ -222,10 +220,6 @@ module Liquid
       render(*args)
     end
 
-    def render_to_output_buffer(context, output)
-      render(context, output: output)
-    end
-
     private
 
     def tokenize(source)

lib/liquid/tokenizer.rb

@@ -1,31 +1,25 @@
 module Liquid
   class Tokenizer
-    attr_reader :line_number, :for_liquid_tag
+    attr_reader :line_number
 
-    def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
+    def initialize(source, line_numbers = false)
       @source = source
-      @line_number = line_number || (line_numbers ? 1 : nil)
+      @line_number = line_numbers ? 1 : nil
-      @for_liquid_tag = for_liquid_tag
       @tokens = tokenize
     end
 
     def shift
-      token = @tokens.shift or return
+      token = @tokens.shift
-
+      @line_number += token.count("\n") if @line_number && token
-      if @line_number
-        @line_number += @for_liquid_tag ? 1 : token.count("\n")
-      end
-
       token
     end
 
     private
 
     def tokenize
+      @source = @source.source if @source.respond_to?(:source)
       return [] if @source.to_s.empty?
 
-      return @source.split("\n") if @for_liquid_tag
-
       tokens = @source.split(TemplateParser)
 
       # removes the rogue empty element at the beginning of the array

lib/liquid/utils.rb

@@ -1,6 +1,8 @@
 module Liquid
   module Utils
     def self.slice_collection(collection, from, to)
+      return collection.load_slice(from, to) if collection.is_a?(ReversableRange)
+
       if (from != 0 || !to.nil?) && collection.respond_to?(:load_slice)
         collection.load_slice(from, to)
       else

lib/liquid/variable.rb

@@ -85,23 +85,12 @@ module Liquid
       end
 
       obj = context.apply_global_filter(obj)
+
       taint_check(context, obj)
+
       obj
     end
 
-    def render_to_output_buffer(context, output)
-      obj = render(context)
-
-      if obj.is_a?(Array)
-        output << obj.join
-      elsif obj.nil?
-      else
-        output << obj.to_s
-      end
-
-      output
-    end
-
     private
 
     def parse_filter_expressions(filter_name, unparsed_args)

performance/memory_profile.rb

@@ -2,61 +2,25 @@
 
 require 'benchmark/ips'
 require 'memory_profiler'
-require 'terminal-table'
 require_relative 'theme_runner'
 
-class Profiler
-  LOG_LABEL   = "Profiling: ".rjust(14).freeze
-  REPORTS_DIR = File.expand_path('.memprof', __dir__).freeze
-
-  def self.run
-    puts
-    yield new
-  end
-
-  def initialize
-    @allocated = []
-    @retained  = []
-    @headings  = []
-  end
-
 def profile(phase, &block)
-    print LOG_LABEL
-    print "#{phase}.. ".ljust(10)
-    report = MemoryProfiler.report(&block)
-    puts 'Done.'
-    @headings  << phase.capitalize
-    @allocated << "#{report.scale_bytes(report.total_allocated_memsize)} (#{report.total_allocated} objects)"
-    @retained  << "#{report.scale_bytes(report.total_retained_memsize)} (#{report.total_retained} objects)"
-
-    return if ENV['CI']
-    require 'fileutils'
-    report_file = File.join(REPORTS_DIR, "#{sanitize(phase)}.txt")
-    FileUtils.mkdir_p(REPORTS_DIR)
-    report.pretty_print(to_file: report_file, scale_bytes: true)
-  end
-
-  def tabulate
-    table = Terminal::Table.new(headings: @headings.unshift('Phase')) do |t|
-      t << @allocated.unshift('Total allocated')
-      t << @retained.unshift('Total retained')
-    end
-
   puts
-    puts table
+  puts "#{phase}:"
-    puts "\nDetailed report(s) saved to #{REPORTS_DIR}/" unless ENV['CI']
+  puts
-  end
 
-  def sanitize(string)
+  report = MemoryProfiler.report(&block)
-    string.downcase.gsub(/[\W]/, '-').squeeze('-')
+
-  end
+  report.pretty_print(
+    color_output: true,
+    scale_bytes: true,
+    detailed_report: true
+  )
 end
 
 Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
 
-runner = ThemeRunner.new
+profiler = ThemeRunner.new
-Profiler.run do |x|
+
-  x.profile('parse') { runner.compile }
+profile("Parsing") { profiler.compile }
-  x.profile('render') { runner.render }
+profile("Rendering") { profiler.render }
-  x.tabulate
-end

performance/shopify/comment_form.rb

@@ -12,7 +12,7 @@ class CommentForm < Liquid::Block
     end
   end
 
-  def render_to_output_buffer(context, output)
+  def render(context)
     article = context[@variable_name]
 
     context.stack do
@@ -23,9 +23,7 @@ class CommentForm < Liquid::Block
         'email'  => context['comment.email'],
         'body'   => context['comment.body']
       }
-
+      wrap_in_form(article, render_all(@nodelist, context))
-      output << wrap_in_form(article, render_all(@nodelist, context, output))
-      output
     end
   end
 

performance/shopify/paginate.rb

@@ -21,7 +21,7 @@ class Paginate < Liquid::Block
     end
   end
 
-  def render_to_output_buffer(context, output)
+  def render(context)
     @context = context
 
     context.stack do

test/integration/blank_test.rb

@@ -1,10 +1,11 @@
 require 'test_helper'
 
 class FoobarTag < Liquid::Tag
-  def render_to_output_buffer(context, output)
+  def render(*args)
-    output << ' '
+    " "
-    output
   end
+
+  Liquid::Template.register_tag('foobar', FoobarTag)
 end
 
 class BlankTestFileSystem
@@ -30,10 +31,8 @@ class BlankTest < Minitest::Test
   end
 
   def test_new_tags_are_not_blank_by_default
-    with_custom_tag('foobar', FoobarTag) do
     assert_template_result(" " * N, wrap_in_for("{% foobar %}"))
   end
-  end
 
   def test_loops_are_blank
     assert_template_result("", wrap_in_for(" "))

test/integration/tags/echo_test.rb

@@ -1,11 +0,0 @@
-require 'test_helper'
-
-class EchoTest < Minitest::Test
-  include Liquid
-
-  def test_echo_outputs_its_input
-    assert_template_result('BAR', <<~LIQUID, { 'variable-name' => 'bar' })
-      {%- echo variable-name | upcase -%}
-    LIQUID
-  end
-end

test/integration/tags/for_tag_test.rb

@@ -36,6 +36,10 @@ HERE
     assert_template_result('321', '{%for item in array reversed %}{{item}}{%endfor%}', assigns)
   end
 
+  def test_for_range_reversed
+    assert_template_result('321', '{%for item in (1..3) reversed %}{{item}}{%endfor%}', {})
+  end
+
   def test_for_with_range
     assert_template_result(' 1  2  3 ', '{%for item in (1..3) %} {{item}} {%endfor%}')
 

test/integration/tags/include_tag_test.rb

@@ -66,9 +66,8 @@ class CustomInclude < Liquid::Tag
   def parse(tokens)
   end
 
-  def render_to_output_buffer(context, output)
+  def render(context)
-    output << @template_name[1..-2]
+    @template_name[1..-2]
-    output
   end
 end
 

test/integration/tags/liquid_tag_test.rb

@@ -1,104 +0,0 @@
-require 'test_helper'
-
-class LiquidTagTest < Minitest::Test
-  include Liquid
-
-  def test_liquid_tag
-    assert_template_result('1 2 3', <<~LIQUID, 'array' => [1, 2, 3])
-    {%- liquid
-      echo array | join: " "
-    -%}
-    LIQUID
-
-    assert_template_result('1 2 3', <<~LIQUID, 'array' => [1, 2, 3])
-    {%- liquid
-      for value in array
-        echo value
-        unless forloop.last
-          echo " "
-        endunless
-      endfor
-    -%}
-    LIQUID
-
-    assert_template_result('4 8 12 6', <<~LIQUID, 'array' => [1, 2, 3])
-    {%- liquid
-      for value in array
-        assign double_value = value | times: 2
-        echo double_value | times: 2
-        unless forloop.last
-          echo " "
-        endunless
-      endfor
-
-      echo " "
-      echo double_value
-    -%}
-    LIQUID
-
-    assert_template_result('abc', <<~LIQUID)
-    {%- liquid echo "a" -%}
-    b
-    {%- liquid echo "c" -%}
-    LIQUID
-  end
-
-  def test_liquid_tag_errors
-    assert_match_syntax_error("syntax error (line 1): Unknown tag 'error'", <<~LIQUID)
-    {%- liquid error no such tag -%}
-    LIQUID
-
-    assert_match_syntax_error("syntax error (line 7): Unknown tag 'error'", <<~LIQUID)
-    {{ test }}
-
-    {%-
-    liquid
-      for value in array
-
-        error no such tag
-      endfor
-    -%}
-    LIQUID
-
-    assert_match_syntax_error("syntax error (line 2): Unknown tag '!!! the guards are vigilant'", <<~LIQUID)
-    {%- liquid
-      !!! the guards are vigilant
-    -%}
-    LIQUID
-
-    assert_match_syntax_error("syntax error (line 4): 'for' tag was never closed", <<~LIQUID)
-    {%- liquid
-      for value in array
-        echo 'forgot to close the for tag'
-    -%}
-    LIQUID
-  end
-
-  def test_line_number_is_correct_after_a_blank_token
-    assert_match_syntax_error("syntax error (line 3): Unknown tag 'error'", "{% liquid echo ''\n\n error %}")
-    assert_match_syntax_error("syntax error (line 3): Unknown tag 'error'", "{% liquid echo ''\n  \n error %}")
-  end
-
-  def test_cannot_open_blocks_living_past_a_liquid_tag
-    assert_match_syntax_error("syntax error (line 3): 'if' tag was never closed", <<~LIQUID)
-    {%- liquid
-      if true
-    -%}
-    {%- endif -%}
-    LIQUID
-  end
-
-  def test_quirk_can_close_blocks_created_before_a_liquid_tag
-    assert_template_result("42", <<~LIQUID)
-    {%- if true -%}
-    42
-    {%- liquid endif -%}
-    LIQUID
-  end
-
-  def test_liquid_tag_in_raw
-    assert_template_result("{% liquid echo 'test' %}\n", <<~LIQUID)
-    {% raw %}{% liquid echo 'test' %}{% endraw %}
-    LIQUID
-  end
-end

test/integration/variable_test.rb

@@ -80,7 +80,10 @@ class VariableTest < Minitest::Test
     assigns['test'] = 'Tobi'
     assert_equal 'Hello Tobi', template.render!(assigns)
     assigns.delete('test')
-    assert_equal "Hello ", template.render!(assigns)
+    e = assert_raises(RuntimeError) do
+      template.render!(assigns)
+    end
+    assert_equal "Unknown variable 'test'", e.message
   end
 
   def test_multiline_variable

test/test_helper.rb

@@ -37,18 +37,18 @@ module Minitest
     include Liquid
 
     def assert_template_result(expected, template, assigns = {}, message = nil)
-      assert_equal expected, Template.parse(template, line_numbers: true).render!(assigns), message
+      assert_equal expected, Template.parse(template).render!(assigns), message
     end
 
     def assert_template_result_matches(expected, template, assigns = {}, message = nil)
       return assert_template_result(expected, template, assigns, message) unless expected.is_a? Regexp
 
-      assert_match expected, Template.parse(template, line_numbers: true).render!(assigns), message
+      assert_match expected, Template.parse(template).render!(assigns), message
     end
 
     def assert_match_syntax_error(match, template, assigns = {})
       exception = assert_raises(Liquid::SyntaxError) do
-        Template.parse(template, line_numbers: true).render(assigns)
+        Template.parse(template).render(assigns)
       end
       assert_match match, exception.message
     end
@@ -84,13 +84,6 @@ module Minitest
     ensure
       Liquid::Template.error_mode = old_mode
     end
-
-    def with_custom_tag(tag_name, tag_class)
-      Liquid::Template.register_tag(tag_name, tag_class)
-      yield
-    ensure
-      Liquid::Template.tags.delete(tag_name)
-    end
   end
 end
 

test/unit/block_unit_test.rb

@@ -44,47 +44,10 @@ class BlockUnitTest < Minitest::Test
   end
 
   def test_with_custom_tag
-    with_custom_tag('testtag', Block) do
+    Liquid::Template.register_tag("testtag", Block)
     assert Liquid::Template.parse("{% testtag %} {% endtesttag %}")
-    end
+  ensure
-  end
+    Liquid::Template.tags.delete('testtag')
-
-  def test_custom_block_tags_have_a_default_render_to_output_buffer_method_for_backwards_compatibility
-    klass1 = Class.new(Block) do
-      def render(*)
-        'hello'
-      end
-    end
-
-    with_custom_tag('blabla', klass1) do
-      template = Liquid::Template.parse("{% blabla %} bla {% endblabla %}")
-
-      assert_equal 'hello', template.render
-
-      buf = ''
-      output = template.render({}, output: buf)
-      assert_equal 'hello', output
-      assert_equal 'hello', buf
-      assert_equal buf.object_id, output.object_id
-    end
-
-    klass2 = Class.new(klass1) do
-      def render(*)
-        'foo' + super + 'bar'
-      end
-    end
-
-    with_custom_tag('blabla', klass2) do
-      template = Liquid::Template.parse("{% blabla %} foo {% endblabla %}")
-
-      assert_equal 'foohellobar', template.render
-
-      buf = ''
-      output = template.render({}, output: buf)
-      assert_equal 'foohellobar', output
-      assert_equal 'foohellobar', buf
-      assert_equal buf.object_id, output.object_id
-    end
   end
 
   private

test/unit/context_unit_test.rb

@@ -349,9 +349,9 @@ class ContextUnitTest < Minitest::Test
 
   def test_ranges
     @context.merge("test" => '5')
-    assert_equal (1..5), @context['(1..5)']
+    assert_equal ReversableRange.new(1, 5), @context['(1..5)']
-    assert_equal (1..5), @context['(1..test)']
+    assert_equal ReversableRange.new(1, 5), @context['(1..test)']
-    assert_equal (5..5), @context['(test..test)']
+    assert_equal ReversableRange.new(5, 5), @context['(test..test)']
   end
 
   def test_cents_through_drop_nestedly

test/unit/reversable_range_unit_test.rb

@@ -0,0 +1,116 @@
+require 'test_helper'
+
+class ReversableRangeTest < Minitest::Test
+  include Liquid
+
+  def test_each_iterates_through_items_in_the_range
+    actual_items = []
+    ReversableRange.new(1, 10).each { |item| actual_items << item }
+
+    expected_items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+    assert_equal expected_items, actual_items
+  end
+
+  def test_implements_enumerable
+    actual_items = ReversableRange.new(1, 10).select(&:even?)
+
+    expected_items = [2, 4, 6, 8, 10]
+    assert_equal expected_items, actual_items
+  end
+
+  def test_is_not_empty_max_greater_than_min
+    range = ReversableRange.new(9, 10)
+
+    refute_predicate range, :empty?
+  end
+
+  def test_is_not_empty_max_equal_to_min
+    range = ReversableRange.new(10, 10)
+
+    refute_predicate range, :empty?
+  end
+
+  def test_is_empty_if_not_reversed_and_max_less_than_min
+    range = ReversableRange.new(10, 9)
+
+    assert_predicate range, :empty?
+  end
+
+  def test_ranges_with_the_same_max_and_min_have_one_element
+    actual_items = ReversableRange.new(1337, 1337).to_a
+    expected_items = [1337]
+    assert_equal expected_items, actual_items
+  end
+
+  def test_load_slice_returns_a_sub_range
+    actual_items = ReversableRange.new(1, 10).load_slice(5, 8).to_a
+
+    expected_items = [5, 6, 7, 8]
+    assert_equal expected_items, actual_items
+  end
+
+  def test_load_slice_returns_a_reversed_sub_range_if_reversed
+    range = ReversableRange.new(1, 10)
+    range.reverse!
+    actual_items = range.load_slice(5, 8).to_a
+
+    expected_items = [8, 7, 6, 5]
+    assert_equal expected_items, actual_items
+  end
+
+  def test_is_equal_to_other_if_also_a_reversable_range_and_has_same_properties
+    one = ReversableRange.new(1, 10)
+    one.reverse!
+
+    two = ReversableRange.new(1, 10)
+    two.reverse!
+
+    assert_equal one, two
+  end
+
+  def test_is_not_equal_to_a_non_reversable_range
+    range = ReversableRange.new(1, 10)
+    range.reverse!
+
+    refute_equal range, :something_else
+  end
+
+  def test_is_not_equal_if_ranges_have_different_mins
+    one = ReversableRange.new(1, 10)
+    two = ReversableRange.new(2, 10)
+
+    refute_equal one, two
+  end
+
+  def test_is_not_equal_if_ranges_have_different_maxes
+    one = ReversableRange.new(1, 10)
+    two = ReversableRange.new(1, 11)
+
+    refute_equal one, two
+  end
+
+  def test_is_not_equal_if_only_one_is_reversed
+    one = ReversableRange.new(1, 10)
+
+    two = ReversableRange.new(1, 10)
+    two.reverse!
+
+    refute_equal one, two
+  end
+
+  def test_to_s_mirrors_rubys_range_syntax
+    range = ReversableRange.new(1, 10)
+    assert_equal '1..10', range.to_s
+  end
+
+  def test_to_s_reverses_when_reversed
+    range = ReversableRange.new(1, 10)
+    range.reverse!
+    assert_equal '10..1', range.to_s
+  end
+
+  def test_size
+    range = ReversableRange.new(1, 10)
+    assert_equal 10, range.size
+  end
+end

test/unit/tag_unit_test.rb

@@ -18,42 +18,4 @@ class TagUnitTest < Minitest::Test
     tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal 'some_tag', tag.tag_name
   end
-
-  def test_custom_tags_have_a_default_render_to_output_buffer_method_for_backwards_compatibility
-    klass1 = Class.new(Tag) do
-      def render(*)
-        'hello'
-      end
-    end
-
-    with_custom_tag('blabla', klass1) do
-      template = Liquid::Template.parse("{% blabla %}")
-
-      assert_equal 'hello', template.render
-
-      buf = ''
-      output = template.render({}, output: buf)
-      assert_equal 'hello', output
-      assert_equal 'hello', buf
-      assert_equal buf.object_id, output.object_id
-    end
-
-    klass2 = Class.new(klass1) do
-      def render(*)
-        'foo' + super + 'bar'
-      end
-    end
-
-    with_custom_tag('blabla', klass2) do
-      template = Liquid::Template.parse("{% blabla %}")
-
-      assert_equal 'foohellobar', template.render
-
-      buf = ''
-      output = template.render({}, output: buf)
-      assert_equal 'foohellobar', output
-      assert_equal 'foohellobar', buf
-      assert_equal buf.object_id, output.object_id
-    end
-  end
 end