WIP

Reduce allocations while registering Liquid tags

.gitignore

@@ -7,3 +7,4 @@ pkg
 .ruby-version
 Gemfile.lock
 .bundle
+.byebug_history

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2019-03-19 11:04:37 -0400 using RuboCop version 0.53.0.
+# on 2019-04-22 19:11:24 -0400 using RuboCop version 0.53.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -46,18 +46,18 @@ Lint/Void:
   Exclude:
     - 'lib/liquid/parse_context.rb'
 
-# Offense count: 54
+# Offense count: 53
 Metrics/AbcSize:
   Max: 56
 
 # Offense count: 12
 Metrics/CyclomaticComplexity:
-  Max: 12
+  Max: 13
 
 # Offense count: 112
 # Configuration parameters: CountComments.
 Metrics/MethodLength:
-  Max: 37
+  Max: 38
 
 # Offense count: 8
 Metrics/PerceivedComplexity:
@@ -90,7 +90,7 @@ Naming/UncommunicativeMethodParamName:
     - 'test/integration/template_test.rb'
     - 'test/unit/condition_unit_test.rb'
 
-# Offense count: 10
+# Offense count: 12
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: prefer_alias, prefer_alias_method
@@ -253,7 +253,7 @@ Style/WhileUntilModifier:
   Exclude:
     - 'lib/liquid/tags/case.rb'
 
-# Offense count: 640
+# Offense count: 648
 # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
 # URISchemes: http, https
 Metrics/LineLength:

Gemfile

@@ -5,6 +5,7 @@ end
 
 gemspec
 
+
 group :benchmark, :test do
   gem 'benchmark-ips'
   gem 'memory_profiler'
@@ -16,6 +17,9 @@ end
 
 group :test do
   gem 'rubocop', '~> 0.53.0'
+  gem 'awesome_print'
+  gem 'pry'
+  gem 'byebug'
 
   platform :mri do
     gem 'liquid-c', github: 'Shopify/liquid-c', ref: '9168659de45d6d576fce30c735f857e597fa26f6'

Rakefile

@@ -45,6 +45,17 @@ task :test do
   end
 end
 
+desc 'runs the test suite using the superfluid compiler'
+Rake::TestTask.new(:test_superfluid) do |t|
+  t.libs << '.' << 'lib' << 'test'
+  t.test_files = FileList['test/integration/**/*_test.rb']
+  t.verbose = false
+
+  ENV['LIQUID_PARSER_MODE'] = 'strict'
+  ENV['LIQUID-C'] = '1'
+  ENV['SUPERFLUID'] = '1'
+end
+
 task gem: :build
 task :build do
   system "gem build liquid.gemspec"
@@ -71,6 +82,11 @@ namespace :benchmark do
   task :strict do
     ruby "./performance/benchmark.rb strict"
   end
+
+  desc "Run the liquid benchmark with strict parsing"
+  task :superfluid do
+    ruby "./performance/benchmark.rb superfluid"
+  end
 end
 
 namespace :profile do

lib/liquid/block.rb

@@ -13,6 +13,7 @@ module Liquid
       end
     end
 
+    # For backwards compatibility
     def render(context)
       @body.render(context)
     end

lib/liquid/block_body.rb

@@ -67,19 +67,23 @@ module Liquid
     end
 
     def render(context)
-      output = []
+      render_to_output_buffer(context, '')
+    end
+
+    def render_to_output_buffer(context, output)
       context.resource_limits.render_score += @nodelist.length
 
       idx = 0
       while node = @nodelist[idx]
+        previous_output_size = output.bytesize
+
         case node
         when String
-          check_resources(context, node)
           output << node
         when Variable
-          render_node_to_output(node, output, context)
+          render_node(context, output, node)
         when Block
-          render_node_to_output(node, output, context, node.blank?)
+          render_node(context, node.blank? ? '' : output, node)
           break if context.interrupt? # might have happened in a for-block
         when Continue, Break
           # If we get an Interrupt that means the block must stop processing. An
@@ -88,38 +92,28 @@ module Liquid
           context.push_interrupt(node.interrupt)
           break
         else # Other non-Block tags
-          render_node_to_output(node, output, context)
+          render_node(context, output, node)
           break if context.interrupt? # might have happened through an include
         end
         idx += 1
+
+        context.raise_if_resource_limits_reached(output.bytesize - previous_output_size)
       end
 
-      output.join
+      output
     end
 
     private
 
-    def render_node_to_output(node, output, context, skip_output = false)
+    def render_node(context, output, node)
-      node_output = node.render(context)
+      node.render_to_output_buffer(context, output)
-      node_output = node_output.is_a?(Array) ? node_output.join : node_output.to_s
-      check_resources(context, node_output)
-      output << node_output unless skip_output
-    rescue MemoryError => e
-      raise e
     rescue UndefinedVariable, UndefinedDropMethod, UndefinedFilter => e
       context.handle_error(e, node.line_number)
-      output << nil
     rescue ::StandardError => e
       line_number = node.is_a?(String) ? nil : node.line_number
       output << context.handle_error(e, line_number)
     end
 
-    def check_resources(context, node_output)
-      context.resource_limits.render_length += node_output.bytesize
-      return unless context.resource_limits.reached?
-      raise MemoryError.new("Memory limits exceeded".freeze)
-    end
-
     def create_variable(token, parse_context)
       token.scan(ContentOfVariable) do |content|
         markup = content.first

lib/liquid/condition.rb

@@ -29,7 +29,7 @@ module Liquid
       @@operators
     end
 
-    attr_reader :attachment, :child_condition
+    attr_reader :attachment, :child_condition, :child_relation
     attr_accessor :left, :operator, :right
 
     def initialize(left = nil, operator = nil, right = nil)
@@ -81,10 +81,6 @@ module Liquid
       "#<Condition #{[@left, @operator, @right].compact.join(' '.freeze)}>"
     end
 
-    protected
-
-    attr_reader :child_relation
-
     private
 
     def equal_variables(left, right)

lib/liquid/context.rb

@@ -37,6 +37,16 @@ module Liquid
       @global_filter = nil
     end
 
+    def raise_argument_error(message)
+      raise Liquid::ArgumentError, message
+    end
+
+    def raise_if_resource_limits_reached(length)
+      resource_limits.render_length += length
+      return unless resource_limits.reached?
+      raise MemoryError.new("Memory limits exceeded".freeze)
+    end
+
     def warnings
       @warnings ||= []
     end

lib/liquid/parser.rb

@@ -44,11 +44,14 @@ module Liquid
       tok[0] == type
     end
 
+    SINGLE_TOKEN_EXPRESSION_TYPES = [:string, :number].freeze
+    private_constant :SINGLE_TOKEN_EXPRESSION_TYPES
+
     def expression
       token = @tokens[@p]
       if token[0] == :id
         variable_signature
-      elsif [:string, :number].include? token[0]
+      elsif SINGLE_TOKEN_EXPRESSION_TYPES.include? token[0]
         consume
       elsif token.first == :open_round
         consume

lib/liquid/profiler/hooks.rb

@@ -1,23 +1,23 @@
 module Liquid
   class BlockBody
-    def render_node_with_profiling(node, output, context, skip_output = false)
+    def render_node_with_profiling(context, output, node)
       Profiler.profile_node_render(node) do
-        render_node_without_profiling(node, output, context, skip_output)
+        render_node_without_profiling(context, output, node)
       end
     end
 
-    alias_method :render_node_without_profiling, :render_node_to_output
+    alias_method :render_node_without_profiling, :render_node
-    alias_method :render_node_to_output, :render_node_with_profiling
+    alias_method :render_node, :render_node_with_profiling
   end
 
   class Include < Tag
-    def render_with_profiling(context)
+    def render_to_output_buffer_with_profiling(context, output)
       Profiler.profile_children(context.evaluate(@template_name_expr).to_s) do
-        render_without_profiling(context)
+        render_to_output_buffer_without_profiling(context, output)
       end
     end
 
-    alias_method :render_without_profiling, :render
+    alias_method :render_to_output_buffer_without_profiling, :render_to_output_buffer
-    alias_method :render, :render_with_profiling
+    alias_method :render_to_output_buffer, :render_to_output_buffer_with_profiling
   end
 end

lib/liquid/range_lookup.rb

@@ -15,6 +15,8 @@ module Liquid
       @end_obj = end_obj
     end
 
+    attr_reader :start_obj, :end_obj
+
     def evaluate(context)
       start_int = to_integer(context.evaluate(@start_obj))
       end_int = to_integer(context.evaluate(@end_obj))

lib/liquid/standardfilters.rb

@@ -79,7 +79,7 @@ module Liquid
       truncate_string_str = truncate_string.to_s
       l = length - truncate_string_str.length
       l = 0 if l < 0
-      input_str.length > length ? input_str[0...l] + truncate_string_str : input_str
+      input_str.length > length ? input_str[0...l].concat(truncate_string_str) : input_str
     end
 
     def truncatewords(input, words = 15, truncate_string = "...".freeze)
@@ -88,7 +88,7 @@ module Liquid
       words = Utils.to_integer(words)
       l = words - 1
       l = 0 if l < 0
-      wordlist.length > l ? wordlist[0..l].join(" ".freeze) + truncate_string.to_s : input
+      wordlist.length > l ? wordlist[0..l].join(" ".freeze).concat(truncate_string.to_s) : input
     end
 
     # Split input string into an array of substrings separated by given pattern.

lib/liquid/superfluid.rb

@@ -0,0 +1,632 @@
+require 'ap'
+
+require 'pry'
+
+require 'stackprof'
+
+AwesomePrint.defaults = {
+  raw: true
+}
+
+module Liquid
+  Liquid::BlockBody.class_eval do
+    def render_to_output_buffer(context, output)
+
+      ruby = Compiler.compile(@nodelist)
+      
+      if false
+        puts
+        puts "--------------------------- GENERATED RUBY -"
+        line_number = 1
+        puts(ruby.lines.map do |line|
+          "#{line_number}\t#{line}".tap { line_number += 1}
+        end)
+        puts "--------------------------- /GENERATED RUBY -"
+      end
+
+      instructions = RubyVM::InstructionSequence.compile(ruby)
+
+      output_io = StringIO.new
+      instructions.eval.call(output_io, context, Condition)
+
+      output << output_io.string
+    end
+  end
+
+  class SuperfluidError < Exception
+  end
+
+  class Output
+    attr_reader :string, :indent_level
+
+    def initialize(initial_indent)
+      @string = ''.dup
+      @indent_level = initial_indent
+      @indent_str = " " * initial_indent * 2
+    end
+
+    def line(string)
+      @string << @indent_str << string << "\n"
+    end
+
+    def echo(string)
+      output << "liquid_out.write(to_output(#{string}))"
+    end
+
+    def indent(&block)
+      output.indent(&block)
+    end
+
+    def indent
+      @indent_level += 1
+      @indent_str = " " * @indent_level * 2
+      yield
+      @indent_level -= 1
+      @indent_str = " " * @indent_level * 2
+    end
+  end
+
+  class Compiler
+    class << self
+      def compile(template)
+        compiler = new
+        compiler.compile(template)
+        compiler.ruby
+      end
+    end
+
+    def initialize
+      @variables = Set.new
+      @output = Output.new(2)
+      @blank = false
+    end
+
+    def ruby
+      [
+        header,
+        @output.string,
+        trailer
+      ].join("\n")
+    end
+
+
+    def header
+      <<~RUBY
+      module Warning
+        def warn(*)
+        end
+      end
+
+      class ForloopDrop
+        def initialize(name, length, parentloop)
+          @name = name
+          @length = length
+          @parentloop = parentloop
+          @index = 0
+        end
+
+        attr_accessor :parentloop
+
+        def [](value)
+          case value
+          when "length"
+            @length
+          when "name"
+            @name
+          when "index"
+            @index + 1
+          when "index0"
+            @index
+          when "rindex"
+            @length - @index
+          when "rindex0"
+            @length - @index - 1
+          when "first"
+            @index == 0
+          when "last"
+            @index == @length - 1
+          when "parentloop"
+            @parentloop
+          end
+        end
+
+        def to_liquid
+          self
+        end
+
+        def key?(*)
+          true
+        end
+
+        private
+
+        def increment!
+          @index += 1
+        end
+      end
+
+      def slice_collection(collection, from, limit)
+        to = if limit.nil?
+          nil
+        else
+          limit + from
+        end
+
+        if (from != 0 || !to.nil?) && collection.respond_to?(:load_slice)
+          collection.load_slice(from, to)
+        else
+          slice_collection_using_each(collection, from, to)
+        end
+      end
+
+      def slice_collection_using_each(collection, from, to)
+        segments = []
+        index = 0
+
+        if collection.is_a?(String)
+          return collection.empty? ? [] : [collection]
+        end
+        return [] unless collection.respond_to?(:each)
+
+        collection.each do |item|
+          if to && to <= index
+            break
+          end
+
+          if from <= index
+            segments << item
+          end
+
+          index += 1
+        end
+
+        segments
+      end
+
+      def apply_operator(left, operator, right)
+        if left.respond_to?(operator) && right.respond_to?(operator) && !left.is_a?(Hash) && !right.is_a?(Hash)
+          begin
+            left.send(operator, right)
+          rescue ::ArgumentError => e
+            raise @context.raise_argument_error(e.message)
+          end
+        end
+      end
+
+      def contains?(left, right)
+        if left && right && left.respond_to?(:include?)
+          right = right.to_s if left.is_a?(String)
+          left.include?(right)
+        else
+          false
+        end
+      end
+
+      class GlobalVariableLookup
+        def method_missing(method_name, *)
+          nil
+        end
+
+        def to_output(value)
+          output = if value.is_a?(Array)
+            value.join
+          elsif value == nil
+          else
+            value.to_s
+          end
+          @context.apply_global_filter(output)
+        end
+
+        def run(liquid_out, context, condition)
+          @condition = condition
+          @context = context
+          @for_offsets = {}
+          @cycle_values = {}
+          @context.registers[:for_stack] = []
+          @context.registers[:cycle] ||= {}
+          @if_changed_last = nil
+          @prev_output_size = 0
+          #{hoisted_variables}
+      RUBY
+    end
+
+    def trailer
+      <<~RUBY
+          end
+        end
+        GlobalVariableLookup.new.method(:run)
+      RUBY
+    end
+
+    def hoisted_variables
+      @variables.map do |variable|
+        normal_name = unvar(variable)
+        "#{variable} = @context.find_variable(#{normal_name.inspect}, raise_on_not_found: false)"
+      end.join("\n")
+    end
+
+    def compile(node)
+      old_blank = @blank
+      @blank = if node.respond_to?(:blank?)
+        node.blank?
+      elsif !(node.is_a?(String) && node =~ /\A\s*\z/)
+        false
+      else
+        @blank
+      end
+
+      case node
+      when Liquid::Document, Liquid::BlockBody
+        node.nodelist.collect(&method(:compile))
+      when Array
+        node.collect(&method(:compile))
+      when Liquid::Variable
+        compile_variable(node)
+      when Liquid::For
+        compile_for(node)
+      when Liquid::If
+        compile_if(node)
+      when Liquid::Ifchanged
+        compile_if_changed(node)
+      when Liquid::Template
+        compile(node.root)
+      when Liquid::Assign
+        compile_assign(node)
+      when Liquid::Case
+        compile_case(node)
+      when Liquid::Capture
+        compile_capture(node)
+      when String
+        compile_echo_literal(node)
+      when Liquid::Break
+        line "break" if @in_loop
+      when Liquid::Continue
+        line "next" if @in_loop
+      when Liquid::Cycle
+        compile_cycle(node)
+      when Liquid::Raw
+        compile_raw(node)
+      when Liquid::Increment
+        compile_increment(node)
+      when Liquid::Decrement
+        compile_decrement(node)
+      when Liquid::Comment
+      else
+        raise SuperfluidError, "Unknown node type #{node.inspect}"
+      end
+
+      @blank = old_blank
+    end
+
+    def compile_for(node)
+      variable_name = node.variable_name
+
+      collection_name = node.collection_name
+
+      iter_target_expr = case collection_name
+      when Liquid::VariableLookup
+        make_variable_lookup_expr(collection_name)
+      when Range
+        collection_name
+      when Liquid::RangeLookup
+        start_expr = make_variable_expr(collection_name.start_obj)
+        end_expr = make_variable_expr(collection_name.end_obj)
+        line "start = #{start_expr}"
+        line "@context.raise_argument_error('bad value for range') unless start.respond_to?(:to_i)"
+        line "start = #{start_expr}.to_i"
+
+        line "finish = #{end_expr}"
+        line "@context.raise_argument_error('bad value for range') unless finish.respond_to?(:to_i)"
+        line "finish = #{end_expr}.to_i"
+
+        "start..finish"
+      when Liquid::Expression::MethodLiteral
+        '[]'
+      else
+        raise SuperfluidError, "Unknown iteration target: #{collection_name.inspect}"
+      end
+
+      from_expr = if node.from == :continue
+        "@for_offsets['#{node.name}'].to_i"
+      elsif node.from
+        make_variable_expr(node.from)
+      else
+        '0'
+      end
+
+      limit_expr = node.limit ? make_variable_expr(node.limit) : 'nil'
+      line "from = #{from_expr}"
+      line "limit = #{limit_expr}"
+      line "@context.raise_argument_error('invalid integer') unless from.is_a?(Integer)"
+      line "@context.raise_argument_error('invalid integer') unless !limit || limit.is_a?(Integer)"
+
+      line "segment = slice_collection(#{iter_target_expr}, from, limit)"
+      line "segment.reverse!" if node.reversed
+
+      forloop = var('forloop')
+      hoist_var('forloop')
+      line "#{forloop} = ForloopDrop.new('#{node.name}', segment.length, #{forloop})"
+
+      line "if segment.any?"
+      indent do
+        line "segment.each do |#{var(variable_name)}|"
+          indent do
+            line "@context['forloop'] = #{forloop}"
+
+
+            old_in_loop = @in_loop
+            compile(node.for_block)
+            @in_loop = old_in_loop
+
+            line "#{forloop}.send(:increment!)"
+          end
+        line "end"
+      end
+      line "else"
+      indent do
+        compile(node.else_block) if node.else_block
+      end
+      line "end"
+
+      line "@for_offsets['#{node.name}'] = from + segment.length"
+      line "#{forloop} = #{forloop}.parentloop"
+    end
+
+    def compile_if(node)
+      if_condition = node.blocks.first
+      line "if #{make_condition_expr(if_condition)}"
+      indent { if_condition.attachment.nodelist.each(&method(:compile)) }
+
+      node.blocks.drop(1).each do |condition|
+        if condition.left != nil
+          line "elsif #{make_condition_expr(condition)}"
+        else
+          line "else"
+        end
+        indent { condition.attachment.nodelist.each(&method(:compile)) }
+      end
+
+      line "end"
+    end
+
+    def compile_if_changed(node)
+      line "if_changed = lambda do |; liquid_out|"
+      indent do
+        line "liquid_out = StringIO.new"
+        node.nodelist.each(&method(:compile))
+        line "liquid_out.string"
+      end
+      line "end.call"
+
+      line "if if_changed != @if_changed_last"
+      indent { echo "if_changed" }
+      line "end"
+
+      line "@if_changed_last = if_changed"
+    end
+
+    def compile_capture(node)
+      line "#{var(node.to)} = lambda do |; liquid_out|"
+      hoist_var(node.to)
+      indent do
+        line "liquid_out = StringIO.new"
+        node.nodelist.each(&method(:compile))
+        line "liquid_out.string"
+      end
+      line "end.call"
+    end
+
+    def make_condition_expr(node)
+      condition = make_sub_condition_expr(node)
+      if node.child_condition
+        "(#{condition} #{node.child_relation} #{make_condition_expr(node.child_condition)})"
+      else
+        condition
+      end
+    end
+
+    def make_sub_condition_expr(node)
+      return make_variable_expr(node.left) unless node.operator
+
+      operator = node.operator
+      operator = "!=" if operator == "<>"
+
+      if operator == "=="
+        if node.left.is_a?(Liquid::Expression::MethodLiteral) &&
+          node.right.is_a?(Liquid::Expression::MethodLiteral)
+          return "false"
+        elsif node.right.is_a?(Liquid::Expression::MethodLiteral)
+          target = make_variable_expr(node.left)
+          message = node.right.method_name.inspect
+          return "#{target}.respond_to?(#{message}) ? #{target}.send(#{message}) : nil"
+        elsif node.left.is_a?(Liquid::Expression::MethodLiteral)
+          target = make_variable_expr(node.right)
+          message = node.left.method_name.inspect
+          return "#{target}.respond_to?(#{message}) ? #{target}.send(#{message}) : nil"
+        end
+      end
+
+      left = make_variable_expr(node.left)
+      right = make_variable_expr(node.right)
+
+      case operator
+      when "contains"
+        "contains?(#{left}, #{right})"
+      else
+        "apply_operator(#{left}, #{operator.inspect}, #{right})"
+      end
+    end
+
+    def compile_case(node)
+      line 'if false' # HACK
+      else_nodes, if_nodes = node.blocks.partition { |n| n.is_a?(Liquid::ElseCondition) }
+      raise SuperfluidError, 'Too many else nodes' if else_nodes.count > 1
+      else_node = else_nodes.first
+
+      if_nodes.each do |condition|
+        left = make_variable_expr(condition.left)
+        right = make_variable_expr(condition.right)
+        line "elsif #{left} #{condition.operator} #{right}"
+        indent { condition.attachment.nodelist.each(&method(:compile)) }
+      end
+
+      if else_node
+        line "else"
+        indent { else_node.attachment.nodelist.each(&method(:compile)) }
+      end
+
+      line "end"
+    end
+
+    def compile_cycle(node)
+      key = node.name
+      key = key.name if key.is_a?(Liquid::VariableLookup)
+
+      line "key = #{key.inspect}"
+      line "iteration = context.registers[:cycle][key].to_i"
+      line "@cycle_values[key] ||= #{node.variables}"
+
+      line "val = @cycle_values[key][iteration]"
+      echo 'val'
+      line "context.registers[:cycle][key] = (iteration + 1) % #{node.variables.size}"
+    end
+
+    def compile_raw(node)
+      echo "#{node.body.inspect}"
+    end
+
+    def compile_increment(node)
+      line "value = context.environments.first[#{node.variable.inspect}] ||= 0"
+      line "@context.environments.first[#{node.variable.inspect}] = value + 1"
+      echo "value"
+    end
+
+    def compile_decrement(node)
+      line "value = context.environments.first[#{node.variable.inspect}] ||= 0"
+      line "value -= 1"
+      line "@context.environments.first[#{node.variable.inspect}] = value"
+      echo "value"
+    end
+
+    def compile_echo_literal(node)
+      echo node.inspect
+    end
+
+    def compile_variable(variable)
+      echo make_variable_expr(variable)
+    end
+
+    def compile_assign(node)
+      from_expr = case node.from
+      when Liquid::Variable
+        make_variable_expr(node.from)
+      else
+        raise SuperfluidError, "Unknown assignment `from`: #{node.from.inspect}"
+      end
+
+      line "#{var(node.to)} = #{from_expr}"
+      hoist_var(node.to)
+    end
+
+    def make_variable_expr(variable)
+      case variable
+      when Liquid::Variable
+        base_expression = case variable.name
+        when TrueClass, FalseClass, Numeric, String
+          variable.name.inspect
+        when Liquid::VariableLookup
+          make_variable_lookup_expr(variable.name)
+        when NilClass, Liquid::Expression::MethodLiteral
+          'nil'
+        else
+          raise SuperfluidError, "Invalid variable name: #{variable.name.inspect}"
+          derp "Bad var name", variable.name
+        end
+        variable.filters.inject(base_expression) do |inner, (filter_name, positional_args, keyword_args)|
+          filter_args = positional_args.map(&method(:make_variable_expr))
+
+          if keyword_args
+             filter_args << "{ " + keyword_args
+              .transform_values(&method(:make_variable_expr))
+              .collect { |(key, value)| "#{key.inspect} => #{value}" }
+              .join(", ") + " }"
+          end
+
+          "context.strainer.invoke(#{filter_name.inspect}, #{inner}, *[#{filter_args.join(", ")}])"
+        end
+      when Liquid::VariableLookup
+        make_variable_lookup_expr(variable)
+      when TrueClass, FalseClass, Numeric, String
+        variable.inspect
+      when NilClass
+        'nil'
+      else
+        raise SuperfluidError, "Unknown expression type: #{variable.inspect}"
+      end
+    end
+
+    def make_variable_lookup_expr(variable_lookup)
+      base_expr = var(variable_lookup.name)
+      hoist_var(variable_lookup.name)
+      return base_expr if variable_lookup.lookups.empty?
+      
+      expr = Output.new(output.indent_level)
+      expr.line "(begin"
+      expr.indent do
+        expr.line "inner = #{base_expr}"
+        variable_lookup.lookups.each_with_index do |lookup, i|
+          lookup_expr = lookup.inspect
+          expr.line "inner = if inner.respond_to?(:[]) && ((inner.respond_to?(:key?) && inner.key?(#{lookup_expr})) || (inner.respond_to?(:fetch) && #{lookup_expr}.is_a?(Integer)))"
+          expr.indent do
+            expr.line "inner[#{lookup_expr}].to_liquid"
+          end
+          if variable_lookup.command_flags & (1 << i) != 0
+            expr.line "elsif inner.respond_to?(#{lookup.inspect})"
+            expr.indent do
+              expr.line "inner.#{lookup}.to_liquid"
+            end
+          end
+          expr.line "end"
+        end
+      end
+      expr.line "end)"
+      expr.string.strip
+    end
+
+    private
+
+    def line(string)
+      output.line(string)
+    end
+
+    def echo(string)
+      unless @blank
+        line "liquid_out.write(to_output(#{string}))"
+      end
+    end
+
+    def indent(&block)
+      output.indent(&block)
+    end
+
+    def var(name)
+      name = name
+        .gsub('_', '__')
+        .gsub('-', '_')
+      "__liquid_#{name}"
+    end
+
+    def unvar(name)
+      name
+        .delete_prefix('__liquid_')
+        .gsub(/([^_])_([^_])/) { "#$1-#$2" }
+        .gsub('__', '_')
+    end
+
+    def hoist_var(name)
+      @variables << var(name)
+    end
+
+    attr_reader :output
+  end
+end

lib/liquid/tag.rb

@@ -36,6 +36,14 @@ module Liquid
       ''.freeze
     end
 
+    # For backwards compatibility with custom tags. In a future release, the semantics
+    # of the `render_to_output_buffer` method will become the default and the `render`
+    # method will be removed.
+    def render_to_output_buffer(context, output)
+      output << render(context)
+      output
+    end
+
     def blank?
       false
     end

lib/liquid/tags/assign.rb

@@ -22,11 +22,11 @@ module Liquid
       end
     end
 
-    def render(context)
+    def render_to_output_buffer(context, output)
       val = @from.render(context)
       context.scopes.last[@to] = val
       context.resource_limits.assign_score += assign_score_of(val)
-      ''.freeze
+      output
     end
 
     def blank?

lib/liquid/tags/capture.rb

@@ -13,6 +13,8 @@ module Liquid
   class Capture < Block
     Syntax = /(#{VariableSignature}+)/o
 
+    attr_reader :to
+
     def initialize(tag_name, markup, options)
       super
       if markup =~ Syntax
@@ -22,16 +24,18 @@ module Liquid
       end
     end
 
-    def render(context)
+    def render_to_output_buffer(context, output)
-      output = super
+      previous_output_size = output.bytesize
+      super
       context.scopes.last[@to] = output
-      context.resource_limits.assign_score += output.bytesize
+      context.resource_limits.assign_score += (output.bytesize - previous_output_size)
-      ''.freeze
+      output
     end
 
     def blank?
       true
     end
+
   end
 
   Template.register_tag('capture'.freeze, Capture)

lib/liquid/tags/case.rb

@@ -38,22 +38,22 @@ module Liquid
       end
     end
 
-    def render(context)
+    def render_to_output_buffer(context, output)
       context.stack do
         execute_else_block = true
 
-        output = ''
         @blocks.each do |block|
           if block.else?
-            return block.attachment.render(context) if execute_else_block
+            block.attachment.render_to_output_buffer(context, output) if execute_else_block
           elsif block.evaluate(context)
             execute_else_block = false
-            output << block.attachment.render(context)
+            block.attachment.render_to_output_buffer(context, output)
           end
         end
+      end
+
       output
     end
-    end
 
     private
 

lib/liquid/tags/comment.rb

@@ -1,7 +1,7 @@
 module Liquid
   class Comment < Block
-    def render(_context)
+    def render_to_output_buffer(_context, output)
-      ''.freeze
+      output
     end
 
     def unknown_tag(_tag, _markup, _tokens)

lib/liquid/tags/cycle.rb

@@ -15,7 +15,7 @@ module Liquid
     SimpleSyntax = /\A#{QuotedFragment}+/o
     NamedSyntax  = /\A(#{QuotedFragment})\s*\:\s*(.*)/om
 
-    attr_reader :variables
+    attr_reader :variables, :name
 
     def initialize(tag_name, markup, options)
       super
@@ -31,18 +31,29 @@ module Liquid
       end
     end
 
-    def render(context)
+    def render_to_output_buffer(context, output)
       context.registers[:cycle] ||= {}
 
       context.stack do
         key = context.evaluate(@name)
         iteration = context.registers[:cycle][key].to_i
-        result = context.evaluate(@variables[iteration])
+
+        val = context.evaluate(@variables[iteration])
+
+        if val.is_a?(Array)
+          val = val.join
+        elsif !val.is_a?(String)
+          val = val.to_s
+        end
+
+        output << val
+
         iteration += 1
         iteration  = 0 if iteration >= @variables.size
         context.registers[:cycle][key] = iteration
-        result
       end
+
+      output
     end
 
     private

lib/liquid/tags/decrement.rb

@@ -23,11 +23,14 @@ module Liquid
       @variable = markup.strip
     end
 
-    def render(context)
+    attr_reader :variable
+
+    def render_to_output_buffer(context, output)
       value = context.environments.first[@variable] ||= 0
       value -= 1
       context.environments.first[@variable] = value
-      value.to_s
+      output << value.to_s
+      output
     end
   end
 

lib/liquid/tags/for.rb

@@ -46,7 +46,7 @@ module Liquid
   class For < Block
     Syntax = /\A(#{VariableSegment}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o
 
-    attr_reader :collection_name, :variable_name, :limit, :from
+    attr_reader :collection_name, :variable_name, :limit, :from, :for_block, :else_block, :name, :reversed
 
     def initialize(tag_name, markup, options)
       super
@@ -70,14 +70,16 @@ module Liquid
       @else_block = BlockBody.new
     end
 
-    def render(context)
+    def render_to_output_buffer(context, output)
       segment = collection_segment(context)
 
       if segment.empty?
-        render_else(context)
+        render_else(context, output)
       else
-        render_segment(context, segment)
+        render_segment(context, output, segment)
       end
+
+      output
     end
 
     protected
@@ -150,12 +152,10 @@ module Liquid
       segment
     end
 
-    def render_segment(context, segment)
+    def render_segment(context, output, segment)
       for_stack = context.registers[:for_stack] ||= []
       length = segment.length
 
-      result = ''
-
       context.stack do
         loop_vars = Liquid::ForloopDrop.new(@name, length, for_stack[-1])
 
@@ -166,7 +166,7 @@ module Liquid
 
           segment.each do |item|
             context[@variable_name] = item
-            result << @for_block.render(context)
+            @for_block.render_to_output_buffer(context, output)
             loop_vars.send(:increment!)
 
             # Handle any interrupts if they exist.
@@ -181,7 +181,7 @@ module Liquid
         end
       end
 
-      result
+      output
     end
 
     def set_attribute(key, expr)
@@ -197,8 +197,12 @@ module Liquid
       end
     end
 
-    def render_else(context)
+    def render_else(context, output)
-      @else_block ? @else_block.render(context) : ''.freeze
+      if @else_block
+        @else_block.render_to_output_buffer(context, output)
+      else
+        output
+      end
     end
 
     class ParseTreeVisitor < Liquid::ParseTreeVisitor

lib/liquid/tags/if.rb

@@ -39,15 +39,16 @@ module Liquid
       end
     end
 
-    def render(context)
+    def render_to_output_buffer(context, output)
       context.stack do
         @blocks.each do |block|
           if block.evaluate(context)
-            return block.attachment.render(context)
+            return block.attachment.render_to_output_buffer(context, output)
           end
         end
-        ''.freeze
       end
+
+      output
     end
 
     private

lib/liquid/tags/ifchanged.rb

@@ -1,16 +1,17 @@
 module Liquid
   class Ifchanged < Block
-    def render(context)
+    def render_to_output_buffer(context, output)
       context.stack do
-        output = super
+        block_output = ''
+        super(context, block_output)
+
+        if block_output != context.registers[:ifchanged]
+          context.registers[:ifchanged] = block_output
+          output << block_output
+        end
+      end
 
-        if output != context.registers[:ifchanged]
-          context.registers[:ifchanged] = output
       output
-        else
-          ''.freeze
-        end
-      end
     end
   end
 

lib/liquid/tags/include.rb

@@ -42,7 +42,7 @@ module Liquid
     def parse(_tokens)
     end
 
-    def render(context)
+    def render_to_output_buffer(context, output)
       template_name = context.evaluate(@template_name_expr)
       raise ArgumentError.new(options[:locale].t("errors.argument.include")) unless template_name
 
@@ -66,19 +66,21 @@ module Liquid
           end
 
           if variable.is_a?(Array)
-            variable.collect do |var|
+            variable.each do |var|
               context[context_variable_name] = var
-              partial.render(context)
+              partial.render_to_output_buffer(context, output)
             end
           else
             context[context_variable_name] = variable
-            partial.render(context)
+            partial.render_to_output_buffer(context, output)
           end
         end
       ensure
         context.template_name = old_template_name
         context.partial = old_partial
       end
+
+      output
     end
 
     private

lib/liquid/tags/increment.rb

@@ -20,10 +20,13 @@ module Liquid
       @variable = markup.strip
     end
 
-    def render(context)
+    attr_reader :variable
+
+    def render_to_output_buffer(context, output)
       value = context.environments.first[@variable] ||= 0
       context.environments.first[@variable] = value + 1
-      value.to_s
+      output << value.to_s
+      output
     end
   end
 

lib/liquid/tags/raw.rb

@@ -3,6 +3,8 @@ module Liquid
     Syntax = /\A\s*\z/
     FullTokenPossiblyInvalid = /\A(.*)#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om
 
+    attr_reader :body
+
     def initialize(tag_name, markup, parse_context)
       super
 
@@ -22,8 +24,9 @@ module Liquid
       raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_never_closed".freeze, block_name: block_name))
     end
 
-    def render(_context)
+    def render_to_output_buffer(_context, output)
-      @body
+      output << @body
+      output
     end
 
     def nodelist

lib/liquid/tags/table_row.rb

@@ -18,7 +18,7 @@ module Liquid
       end
     end
 
-    def render(context)
+    def render_to_output_buffer(context, output)
       collection = context.evaluate(@collection_name) or return ''.freeze
 
       from = @attributes.key?('offset'.freeze) ? context.evaluate(@attributes['offset'.freeze]).to_i : 0
@@ -30,7 +30,7 @@ module Liquid
 
       cols = context.evaluate(@attributes['cols'.freeze]).to_i
 
-      result = "<tr class=\"row1\">\n"
+      output << "<tr class=\"row1\">\n"
       context.stack do
         tablerowloop = Liquid::TablerowloopDrop.new(length, cols)
         context['tablerowloop'.freeze] = tablerowloop
@@ -38,17 +38,20 @@ module Liquid
         collection.each do |item|
           context[@variable_name] = item
 
-          result << "<td class=\"col#{tablerowloop.col}\">" << super << '</td>'
+          output << "<td class=\"col#{tablerowloop.col}\">"
+          super
+          output << '</td>'
 
           if tablerowloop.col_last && !tablerowloop.last
-            result << "</tr>\n<tr class=\"row#{tablerowloop.row + 1}\">"
+            output << "</tr>\n<tr class=\"row#{tablerowloop.row + 1}\">"
           end
 
           tablerowloop.send(:increment!)
         end
       end
-      result << "</tr>\n"
+
-      result
+      output << "</tr>\n"
+      output
     end
 
     class ParseTreeVisitor < Liquid::ParseTreeVisitor

lib/liquid/tags/unless.rb

@@ -6,23 +6,23 @@ module Liquid
   #   {% unless x < 0 %} x is greater than zero {% endunless %}
   #
   class Unless < If
-    def render(context)
+    def render_to_output_buffer(context, output)
       context.stack do
         # First condition is interpreted backwards ( if not )
         first_block = @blocks.first
         unless first_block.evaluate(context)
-          return first_block.attachment.render(context)
+          return first_block.attachment.render_to_output_buffer(context, output)
         end
 
         # After the first condition unless works just like if
         @blocks[1..-1].each do |block|
           if block.evaluate(context)
-            return block.attachment.render(context)
+            return block.attachment.render_to_output_buffer(context, output)
+          end
         end
       end
 
-        ''.freeze
+      output
-      end
     end
   end
 

lib/liquid/template.rb

@@ -50,7 +50,7 @@ module Liquid
       private
 
       def lookup_class(name)
-        name.split("::").reject(&:empty?).reduce(Object) { |scope, const| scope.const_get(const) }
+        Object.const_get(name)
       end
     end
 
@@ -187,9 +187,12 @@ module Liquid
         raise ArgumentError, "Expected Hash or Liquid::Context as parameter"
       end
 
+      output = nil
+
       case args.last
       when Hash
         options = args.pop
+        output = options[:output] if options[:output]
 
         registers.merge!(options[:registers]) if options[:registers].is_a?(Hash)
 
@@ -203,11 +206,10 @@ module Liquid
 
       begin
         # render the nodelist.
-        # for performance reasons we get an array back here. join will make a string out of it.
+
-        result = with_profiling(context) do
+        with_profiling(context) do
-          @root.render(context)
+          @root.render_to_output_buffer(context, output || '')
         end
-        result.respond_to?(:join) ? result.join : result
       rescue Liquid::MemoryError => e
         context.handle_error(e)
       ensure
@@ -220,6 +222,10 @@ module Liquid
       render(*args)
     end
 
+    def render_to_output_buffer(context, output)
+      render(context, output: output)
+    end
+
     private
 
     def tokenize(source)

lib/liquid/variable.rb

@@ -85,12 +85,23 @@ module Liquid
       end
 
       obj = context.apply_global_filter(obj)
-
       taint_check(context, obj)
-
       obj
     end
 
+    def render_to_output_buffer(context, output)
+      obj = render(context)
+
+      if obj.is_a?(Array)
+        output << obj.join
+      elsif obj.nil?
+      else
+        output << obj.to_s
+      end
+
+      output
+    end
+
     private
 
     def parse_filter_expressions(filter_name, unparsed_args)

lib/liquid/variable_lookup.rb

@@ -3,7 +3,7 @@ module Liquid
     SQUARE_BRACKETED = /\A\[(.*)\]\z/m
     COMMAND_METHODS = ['size'.freeze, 'first'.freeze, 'last'.freeze].freeze
 
-    attr_reader :name, :lookups
+    attr_reader :name, :lookups, :command_flags
 
     def self.parse(markup)
       new(markup)

performance/benchmark.rb

@@ -1,7 +1,16 @@
 require 'benchmark/ips'
 require_relative 'theme_runner'
 
+case ARGV.first.to_sym
+when :lax
   Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
+when :strict
+  Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
+when :superfluid
+  require 'liquid/superfluid'
+  Liquid::Template.error_mode = :strict
+end
+
 profiler = ThemeRunner.new
 
 Benchmark.ips do |x|

performance/memory_profile.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'benchmark/ips'
+requirf 'benchmark/ips'
 require 'memory_profiler'
 require_relative 'theme_runner'
 

performance/shopify/comment_form.rb

@@ -12,7 +12,7 @@ class CommentForm < Liquid::Block
     end
   end
 
-  def render(context)
+  def render_to_output_buffer(context, output)
     article = context[@variable_name]
 
     context.stack do
@@ -23,7 +23,9 @@ class CommentForm < Liquid::Block
         'email'  => context['comment.email'],
         'body'   => context['comment.body']
       }
-      wrap_in_form(article, render_all(@nodelist, context))
+
+      output << wrap_in_form(article, render_all(@nodelist, context, output))
+      output
     end
   end
 

performance/shopify/paginate.rb

@@ -21,7 +21,7 @@ class Paginate < Liquid::Block
     end
   end
 
-  def render(context)
+  def render_to_output_buffer(context, output)
     @context = context
 
     context.stack do

test/integration/blank_test.rb

@@ -1,11 +1,10 @@
 require 'test_helper'
 
 class FoobarTag < Liquid::Tag
-  def render(*args)
+  def render_to_output_buffer(context, output)
-    " "
+    output << ' '
+    output
   end
-
-  Liquid::Template.register_tag('foobar', FoobarTag)
 end
 
 class BlankTestFileSystem
@@ -31,8 +30,10 @@ class BlankTest < Minitest::Test
   end
 
   def test_new_tags_are_not_blank_by_default
+    with_custom_tag('foobar', FoobarTag) do
       assert_template_result(" " * N, wrap_in_for("{% foobar %}"))
     end
+  end
 
   def test_loops_are_blank
     assert_template_result("", wrap_in_for(" "))

test/integration/tags/include_tag_test.rb

@@ -66,8 +66,9 @@ class CustomInclude < Liquid::Tag
   def parse(tokens)
   end
 
-  def render(context)
+  def render_to_output_buffer(context, output)
-    @template_name[1..-2]
+    output << @template_name[1..-2]
+    output
   end
 end
 

test/test_helper.rb

@@ -19,6 +19,11 @@ if ENV['LIQUID-C'] == '1'
   require 'liquid/c'
 end
 
+if ENV['SUPERFLUID'] == '1'
+  puts "-- SUPERFLUID"
+  require 'liquid/superfluid'
+end
+
 if Minitest.const_defined?('Test')
   # We're on Minitest 5+. Nothing to do here.
 else
@@ -84,6 +89,13 @@ module Minitest
     ensure
       Liquid::Template.error_mode = old_mode
     end
+
+    def with_custom_tag(tag_name, tag_class)
+      Liquid::Template.register_tag(tag_name, tag_class)
+      yield
+    ensure
+      Liquid::Template.tags.delete(tag_name)
+    end
   end
 end
 

test/unit/block_unit_test.rb

@@ -44,10 +44,47 @@ class BlockUnitTest < Minitest::Test
   end
 
   def test_with_custom_tag
-    Liquid::Template.register_tag("testtag", Block)
+    with_custom_tag('testtag', Block) do
       assert Liquid::Template.parse("{% testtag %} {% endtesttag %}")
-  ensure
+    end
-    Liquid::Template.tags.delete('testtag')
+  end
+
+  def test_custom_block_tags_have_a_default_render_to_output_buffer_method_for_backwards_compatibility
+    klass1 = Class.new(Block) do
+      def render(*)
+        'hello'
+      end
+    end
+
+    with_custom_tag('blabla', klass1) do
+      template = Liquid::Template.parse("{% blabla %} bla {% endblabla %}")
+
+      assert_equal 'hello', template.render
+
+      buf = ''
+      output = template.render({}, output: buf)
+      assert_equal 'hello', output
+      assert_equal 'hello', buf
+      assert_equal buf.object_id, output.object_id
+    end
+
+    klass2 = Class.new(klass1) do
+      def render(*)
+        'foo' + super + 'bar'
+      end
+    end
+
+    with_custom_tag('blabla', klass2) do
+      template = Liquid::Template.parse("{% blabla %} foo {% endblabla %}")
+
+      assert_equal 'foohellobar', template.render
+
+      buf = ''
+      output = template.render({}, output: buf)
+      assert_equal 'foohellobar', output
+      assert_equal 'foohellobar', buf
+      assert_equal buf.object_id, output.object_id
+    end
   end
 
   private

test/unit/tag_unit_test.rb

@@ -18,4 +18,42 @@ class TagUnitTest < Minitest::Test
     tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
     assert_equal 'some_tag', tag.tag_name
   end
+
+  def test_custom_tags_have_a_default_render_to_output_buffer_method_for_backwards_compatibility
+    klass1 = Class.new(Tag) do
+      def render(*)
+        'hello'
+      end
+    end
+
+    with_custom_tag('blabla', klass1) do
+      template = Liquid::Template.parse("{% blabla %}")
+
+      assert_equal 'hello', template.render
+
+      buf = ''
+      output = template.render({}, output: buf)
+      assert_equal 'hello', output
+      assert_equal 'hello', buf
+      assert_equal buf.object_id, output.object_id
+    end
+
+    klass2 = Class.new(klass1) do
+      def render(*)
+        'foo' + super + 'bar'
+      end
+    end
+
+    with_custom_tag('blabla', klass2) do
+      template = Liquid::Template.parse("{% blabla %}")
+
+      assert_equal 'foohellobar', template.render
+
+      buf = ''
+      output = template.render({}, output: buf)
+      assert_equal 'foohellobar', output
+      assert_equal 'foohellobar', buf
+      assert_equal buf.object_id, output.object_id
+    end
+  end
 end