Use same approach in scope and environment

Add a CI job to profile memory usage of commit

.travis.yml

@@ -6,18 +6,21 @@ rvm:
   - 2.3
   - 2.4
   - 2.5
+  - &latest_ruby 2.6
   - ruby-head
   - jruby-head
 #  - rbx-2
 
-sudo: false
-
 addons:
   apt:
     packages:
     - libgmp3-dev
 
 matrix:
+  include:
+    - rvm: *latest_ruby
+      script: bundle exec rake memory_profile:run
+      name: Profiling Memory Usage
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head

Gemfile

@@ -8,6 +8,7 @@ gemspec
 group :benchmark, :test do
   gem 'benchmark-ips'
   gem 'memory_profiler'
+  gem 'terminal-table'
 
   install_if -> { RUBY_PLATFORM !~ /mingw|mswin|java/ } do
     gem 'stackprof'
@@ -18,6 +19,6 @@ group :test do
   gem 'rubocop', '~> 0.53.0'
 
   platform :mri do
-    gem 'liquid-c', github: 'Shopify/liquid-c', ref: '9168659de45d6d576fce30c735f857e597fa26f6'
+    gem 'liquid-c', github: 'Shopify/liquid-c', ref: 'liquid-tag'
   end
 end

lib/liquid/block_body.rb

@@ -1,6 +1,7 @@
 module Liquid
   class BlockBody
-    FullToken = /\A#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
+    LiquidTagToken = /\A\s*(\w+)\s*(.*?)\z/o
+    FullToken = /\A#{TagStart}#{WhitespaceControl}?(\s*)(\w+)(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
     ContentOfVariable = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
     WhitespaceOrNothing = /\A\s*\z/
     TAGSTART = "{%".freeze
@@ -13,8 +14,42 @@ module Liquid
       @blank = true
     end
 
-    def parse(tokenizer, parse_context)
+    def parse(tokenizer, parse_context, &block)
       parse_context.line_number = tokenizer.line_number
+
+      if tokenizer.for_liquid_tag
+        parse_for_liquid_tag(tokenizer, parse_context, &block)
+      else
+        parse_for_document(tokenizer, parse_context, &block)
+      end
+    end
+
+    private def parse_for_liquid_tag(tokenizer, parse_context)
+      while token = tokenizer.shift
+        unless token.empty? || token =~ WhitespaceOrNothing
+          unless token =~ LiquidTagToken
+            # line isn't empty but didn't match tag syntax, yield and let the
+            # caller raise a syntax error
+            return yield token, token
+          end
+          tag_name = $1
+          markup = $2
+          unless tag = registered_tags[tag_name]
+            # end parsing if we reach an unknown tag and let the caller decide
+            # determine how to proceed
+            return yield tag_name, markup
+          end
+          new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
+          @blank &&= new_tag.blank?
+          @nodelist << new_tag
+        end
+        parse_context.line_number = tokenizer.line_number
+      end
+
+      yield nil, nil
+    end
+
+    private def parse_for_document(tokenizer, parse_context, &block)
       while token = tokenizer.shift
         next if token.empty?
         case
@@ -23,9 +58,20 @@ module Liquid
           unless token =~ FullToken
             raise_missing_tag_terminator(token, parse_context)
           end
-          tag_name = $1
+          tag_name = $2
-          markup = $2
+          markup = $4
-          # fetch the tag from registered blocks
+
+          if parse_context.line_number
+            # newlines inside the tag should increase the line number,
+            # particularly important for multiline {% liquid %} tags
+            parse_context.line_number += $1.count("\n".freeze) + $3.count("\n".freeze)
+          end
+
+          if tag_name == 'liquid'.freeze
+            liquid_tag_tokenizer = Tokenizer.new(markup, line_number: parse_context.line_number, for_liquid_tag: true)
+            next parse_for_liquid_tag(liquid_tag_tokenizer, parse_context, &block)
+          end
+
           unless tag = registered_tags[tag_name]
             # end parsing if we reach an unknown tag and let the caller decide
             # determine how to proceed

lib/liquid/context.rb

@@ -20,21 +20,17 @@ module Liquid
       @scopes           = [(outer_scope || {})]
       @registers        = registers
       @errors           = []
+      @interrupts       = []
+      @filters          = []
+      @global_filter    = nil
       @partial          = false
       @strict_variables = false
       @resource_limits  = resource_limits || ResourceLimits.new(Template.default_resource_limits)
-      squash_instance_assigns_with_environments
-
-      @this_stack_used = false
 
       self.exception_renderer = Template.default_exception_renderer
-      if rethrow_errors
+      self.exception_renderer = ->(e) { raise } if rethrow_errors
-        self.exception_renderer = ->(e) { raise }
-      end
 
-      @interrupts = []
+      squash_instance_assigns_with_environments
-      @filters = []
-      @global_filter = nil
     end
 
     def warnings
@@ -87,9 +83,9 @@ module Liquid
     end
 
     # Push new local scope on the stack. use <tt>Context#stack</tt> instead
-    def push(new_scope = {})
+    def push
-      @scopes.unshift(new_scope)
+      @scopes.unshift({})
-      raise StackLevelError, "Nesting too deep".freeze if @scopes.length > Block::MAX_DEPTH
+      raise StackLevelError, "Nesting too deep".freeze if @scopes.length > (Block::MAX_DEPTH + 1)
     end
 
     # Merge a hash of variables in the current local scope
@@ -111,31 +107,15 @@ module Liquid
     #   end
     #
     #   context['var]  #=> nil
-    def stack(new_scope = nil)
+    def stack
-      old_stack_used = @this_stack_used
+      push
-      if new_scope
-        push(new_scope)
-        @this_stack_used = true
-      else
-        @this_stack_used = false
-      end
-
       yield
     ensure
-      pop if @this_stack_used
+      pop
-      @this_stack_used = old_stack_used
-    end
-
-    def clear_instance_assigns
-      @scopes[0] = {}
     end
 
     # Only allow String, Numeric, Hash, Array, Proc, Boolean or <tt>Liquid::Drop</tt>
     def []=(key, value)
-      unless @this_stack_used
-        @this_stack_used = true
-        push({})
-      end
       @scopes[0][key] = value
     end
 
@@ -163,27 +143,12 @@ module Liquid
     def find_variable(key, raise_on_not_found: true)
       # This was changed from find() to find_index() because this is a very hot
       # path and find_index() is optimized in MRI to reduce object allocation
-      index = @scopes.find_index { |s| s.key?(key) }
-      scope = @scopes[index] if index
 
-      variable = nil
+      scope = (index = @scopes.find_index { |s| s.key?(key)  }) && @scopes[index]
+      scope ||= (index = @environments.find_index { |s| s.key?(key) }) && @environments[index]
+      scope ||= {}
 
-      if scope.nil?
+      variable = lookup_and_evaluate(scope, key, raise_on_not_found: raise_on_not_found).to_liquid
-        @environments.each do |e|
-          variable = lookup_and_evaluate(e, key, raise_on_not_found: raise_on_not_found)
-          # When lookup returned a value OR there is no value but the lookup also did not raise
-          # then it is the value we are looking for.
-          if !variable.nil? || @strict_variables && raise_on_not_found
-            scope = e
-            break
-          end
-        end
-      end
-
-      scope ||= @environments.last || @scopes.last
-      variable ||= lookup_and_evaluate(scope, key, raise_on_not_found: raise_on_not_found)
-
-      variable = variable.to_liquid
       variable.context = self if variable.respond_to?(:context=)
 
       variable

lib/liquid/tag.rb

@@ -5,8 +5,8 @@ module Liquid
     include ParserSwitching
 
     class << self
-      def parse(tag_name, markup, tokenizer, options)
+      def parse(tag_name, markup, tokenizer, parse_context)
-        tag = new(tag_name, markup, options)
+        tag = new(tag_name, markup, parse_context)
         tag.parse(tokenizer)
         tag
       end

lib/liquid/tags/assign.rb

@@ -10,6 +10,10 @@ module Liquid
   class Assign < Tag
     Syntax = /(#{VariableSignature}+)\s*=\s*(.*)\s*/om
 
+    def self.syntax_error_translation_key
+      "errors.syntax.assign".freeze
+    end
+
     attr_reader :to, :from
 
     def initialize(tag_name, markup, options)
@@ -18,7 +22,7 @@ module Liquid
         @to = $1
         @from = Variable.new($2, options)
       else
-        raise SyntaxError.new options[:locale].t("errors.syntax.assign".freeze)
+        raise SyntaxError.new(options[:locale].t(self.class.syntax_error_translation_key))
       end
     end
 

lib/liquid/tags/echo.rb

@@ -0,0 +1,24 @@
+module Liquid
+  # Echo outputs an expression
+  #
+  #   {% echo monkey %}
+  #   {% echo user.name %}
+  #
+  # This is identical to variable output syntax, like {{ foo }}, but works
+  # inside {% liquid %} tags. The full syntax is supported, including filters:
+  #
+  #   {% echo user | link %}
+  #
+  class Echo < Tag
+    def initialize(tag_name, markup, parse_context)
+      super
+      @variable = Variable.new(markup, parse_context)
+    end
+
+    def render(context)
+      @variable.render(context)
+    end
+  end
+
+  Template.register_tag('echo'.freeze, Echo)
+end

lib/liquid/tokenizer.rb

@@ -1,25 +1,31 @@
 module Liquid
   class Tokenizer
-    attr_reader :line_number
+    attr_reader :line_number, :for_liquid_tag
 
-    def initialize(source, line_numbers = false)
+    def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
       @source = source
-      @line_number = line_numbers ? 1 : nil
+      @line_number = line_number || (line_numbers ? 1 : nil)
+      @for_liquid_tag = for_liquid_tag
       @tokens = tokenize
     end
 
     def shift
-      token = @tokens.shift
+      token = @tokens.shift or return
-      @line_number += token.count("\n") if @line_number && token
+
+      if @line_number
+        @line_number += @for_liquid_tag ? 1 : token.count("\n")
+      end
+
       token
     end
 
     private
 
     def tokenize
-      @source = @source.source if @source.respond_to?(:source)
       return [] if @source.to_s.empty?
 
+      return @source.split("\n") if @for_liquid_tag
+
       tokens = @source.split(TemplateParser)
 
       # removes the rogue empty element at the beginning of the array

performance/memory_profile.rb

@@ -2,25 +2,61 @@
 
 require 'benchmark/ips'
 require 'memory_profiler'
+require 'terminal-table'
 require_relative 'theme_runner'
 
-def profile(phase, &block)
+class Profiler
-  puts
+  LOG_LABEL   = "Profiling: ".rjust(14).freeze
-  puts "#{phase}:"
+  REPORTS_DIR = File.expand_path('.memprof', __dir__).freeze
-  puts
 
-  report = MemoryProfiler.report(&block)
+  def self.run
+    puts
+    yield new
+  end
 
-  report.pretty_print(
+  def initialize
-    color_output: true,
+    @allocated = []
-    scale_bytes: true,
+    @retained  = []
-    detailed_report: true
+    @headings  = []
-  )
+  end
+
+  def profile(phase, &block)
+    print LOG_LABEL
+    print "#{phase}.. ".ljust(10)
+    report = MemoryProfiler.report(&block)
+    puts 'Done.'
+    @headings  << phase.capitalize
+    @allocated << "#{report.scale_bytes(report.total_allocated_memsize)} (#{report.total_allocated} objects)"
+    @retained  << "#{report.scale_bytes(report.total_retained_memsize)} (#{report.total_retained} objects)"
+
+    return if ENV['CI']
+    require 'fileutils'
+    report_file = File.join(REPORTS_DIR, "#{sanitize(phase)}.txt")
+    FileUtils.mkdir_p(REPORTS_DIR)
+    report.pretty_print(to_file: report_file, scale_bytes: true)
+  end
+
+  def tabulate
+    table = Terminal::Table.new(headings: @headings.unshift('Phase')) do |t|
+      t << @allocated.unshift('Total allocated')
+      t << @retained.unshift('Total retained')
+    end
+
+    puts
+    puts table
+    puts "\nDetailed report(s) saved to #{REPORTS_DIR}/" unless ENV['CI']
+  end
+
+  def sanitize(string)
+    string.downcase.gsub(/[\W]/, '-').squeeze('-')
+  end
 end
 
 Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
 
-profiler = ThemeRunner.new
+runner = ThemeRunner.new
-
+Profiler.run do |x|
-profile("Parsing") { profiler.compile }
+  x.profile('parse') { runner.compile }
-profile("Rendering") { profiler.render }
+  x.profile('render') { runner.render }
+  x.tabulate
+end

test/integration/tags/echo_test.rb

@@ -0,0 +1,11 @@
+require 'test_helper'
+
+class EchoTest < Minitest::Test
+  include Liquid
+
+  def test_echo_outputs_its_input
+    assert_template_result('BAR', <<~LIQUID, { 'variable-name' => 'bar' })
+      {%- echo variable-name | upcase -%}
+    LIQUID
+  end
+end

test/integration/tags/liquid_tag_test.rb

@@ -0,0 +1,104 @@
+require 'test_helper'
+
+class LiquidTagTest < Minitest::Test
+  include Liquid
+
+  def test_liquid_tag
+    assert_template_result('1 2 3', <<~LIQUID, 'array' => [1, 2, 3])
+    {%- liquid
+      echo array | join: " "
+    -%}
+    LIQUID
+
+    assert_template_result('1 2 3', <<~LIQUID, 'array' => [1, 2, 3])
+    {%- liquid
+      for value in array
+        echo value
+        unless forloop.last
+          echo " "
+        endunless
+      endfor
+    -%}
+    LIQUID
+
+    assert_template_result('4 8 12 6', <<~LIQUID, 'array' => [1, 2, 3])
+    {%- liquid
+      for value in array
+        assign double_value = value | times: 2
+        echo double_value | times: 2
+        unless forloop.last
+          echo " "
+        endunless
+      endfor
+
+      echo " "
+      echo double_value
+    -%}
+    LIQUID
+
+    assert_template_result('abc', <<~LIQUID)
+    {%- liquid echo "a" -%}
+    b
+    {%- liquid echo "c" -%}
+    LIQUID
+  end
+
+  def test_liquid_tag_errors
+    assert_match_syntax_error("syntax error (line 1): Unknown tag 'error'", <<~LIQUID)
+    {%- liquid error no such tag -%}
+    LIQUID
+
+    assert_match_syntax_error("syntax error (line 7): Unknown tag 'error'", <<~LIQUID)
+    {{ test }}
+
+    {%-
+    liquid
+      for value in array
+
+        error no such tag
+      endfor
+    -%}
+    LIQUID
+
+    assert_match_syntax_error("syntax error (line 2): Unknown tag '!!! the guards are vigilant'", <<~LIQUID)
+    {%- liquid
+      !!! the guards are vigilant
+    -%}
+    LIQUID
+
+    assert_match_syntax_error("syntax error (line 4): 'for' tag was never closed", <<~LIQUID)
+    {%- liquid
+      for value in array
+        echo 'forgot to close the for tag'
+    -%}
+    LIQUID
+  end
+
+  def test_line_number_is_correct_after_a_blank_token
+    assert_match_syntax_error("syntax error (line 3): Unknown tag 'error'", "{% liquid echo ''\n\n error %}")
+    assert_match_syntax_error("syntax error (line 3): Unknown tag 'error'", "{% liquid echo ''\n  \n error %}")
+  end
+
+  def test_cannot_open_blocks_living_past_a_liquid_tag
+    assert_match_syntax_error("syntax error (line 3): 'if' tag was never closed", <<~LIQUID)
+    {%- liquid
+      if true
+    -%}
+    {%- endif -%}
+    LIQUID
+  end
+
+  def test_quirk_can_close_blocks_created_before_a_liquid_tag
+    assert_template_result("42", <<~LIQUID)
+    {%- if true -%}
+    42
+    {%- liquid endif -%}
+    LIQUID
+  end
+
+  def test_liquid_tag_in_raw
+    assert_template_result("{% liquid echo 'test' %}\n", <<~LIQUID)
+    {% raw %}{% liquid echo 'test' %}{% endraw %}
+    LIQUID
+  end
+end

test/integration/variable_test.rb

@@ -80,10 +80,7 @@ class VariableTest < Minitest::Test
     assigns['test'] = 'Tobi'
     assert_equal 'Hello Tobi', template.render!(assigns)
     assigns.delete('test')
-    e = assert_raises(RuntimeError) do
+    assert_equal "Hello ", template.render!(assigns)
-      template.render!(assigns)
-    end
-    assert_equal "Unknown variable 'test'", e.message
   end
 
   def test_multiline_variable

test/test_helper.rb

@@ -37,18 +37,18 @@ module Minitest
     include Liquid
 
     def assert_template_result(expected, template, assigns = {}, message = nil)
-      assert_equal expected, Template.parse(template).render!(assigns), message
+      assert_equal expected, Template.parse(template, line_numbers: true).render!(assigns), message
     end
 
     def assert_template_result_matches(expected, template, assigns = {}, message = nil)
       return assert_template_result(expected, template, assigns, message) unless expected.is_a? Regexp
 
-      assert_match expected, Template.parse(template).render!(assigns), message
+      assert_match expected, Template.parse(template, line_numbers: true).render!(assigns), message
     end
 
     def assert_match_syntax_error(match, template, assigns = {})
       exception = assert_raises(Liquid::SyntaxError) do
-        Template.parse(template).render(assigns)
+        Template.parse(template, line_numbers: true).render(assigns)
       end
       assert_match match, exception.message
     end