Add {% liquid %}, {% echo %}, and {% local %} tags

Gemfile

@@ -15,6 +15,6 @@ group :test do
   gem 'rubocop', '~> 0.49.0'
 
   platform :mri do
-    gem 'liquid-c', github: 'Shopify/liquid-c', ref: '9168659de45d6d576fce30c735f857e597fa26f6'
+    gem 'liquid-c', github: 'Shopify/liquid-c', ref: 'liquid-tag'
   end
 end

lib/liquid/block_body.rb

@@ -1,6 +1,7 @@
 module Liquid
   class BlockBody
-    FullToken = /\A#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
+    LiquidTagToken = /\A\s*(\w+)\s*(.*?)\z/o
+    FullToken = /\A#{TagStart}#{WhitespaceControl}?(\s*)(\w+)(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
     ContentOfVariable = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
     WhitespaceOrNothing = /\A\s*\z/
     TAGSTART = "{%".freeze
@@ -13,8 +14,46 @@ module Liquid
       @blank = true
     end
 
-    def parse(tokenizer, parse_context)
+    def parse(tokenizer, parse_context, &block)
       parse_context.line_number = tokenizer.line_number
+
+      if tokenizer.for_liquid_tag
+        parse_for_liquid_tag(tokenizer, parse_context, &block)
+      else
+        parse_for_document(tokenizer, parse_context, &block)
+      end
+    end
+
+    private def parse_for_liquid_tag(tokenizer, parse_context)
+      while token = tokenizer.shift
+        case
+        when token.empty?
+          # pass
+        else
+          unless token =~ LiquidTagToken
+            next if token =~ WhitespaceOrNothing
+            # line isn't empty but didn't match tag syntax, yield and let the
+            # caller raise a syntax error
+            return yield token, token
+          end
+          tag_name = $1
+          markup = $2
+          unless tag = registered_tags[tag_name]
+            # end parsing if we reach an unknown tag and let the caller decide
+            # determine how to proceed
+            return yield tag_name, markup
+          end
+          new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
+          @blank &&= new_tag.blank?
+          @nodelist << new_tag
+        end
+        parse_context.line_number = tokenizer.line_number
+      end
+
+      yield nil, nil
+    end
+
+    private def parse_for_document(tokenizer, parse_context, &block)
       while token = tokenizer.shift
         next if token.empty?
         case
@@ -23,9 +62,20 @@ module Liquid
           unless token =~ FullToken
             raise_missing_tag_terminator(token, parse_context)
           end
-          tag_name = $1
-          markup = $2
-          # fetch the tag from registered blocks
+          tag_name = $2
+          markup = $4
+
+          if parse_context.line_number
+            # newlines inside the tag should increase the line number,
+            # particularly important for multiline {% liquid %} tags
+            parse_context.line_number += $1.count("\n".freeze) + $3.count("\n".freeze)
+          end
+
+          if tag_name == 'liquid'.freeze
+            liquid_tag_tokenizer = Tokenizer.new(markup, line_number: parse_context.line_number, for_liquid_tag: true)
+            next parse(liquid_tag_tokenizer, parse_context, &block)
+          end
+
           unless tag = registered_tags[tag_name]
             # end parsing if we reach an unknown tag and let the caller decide
             # determine how to proceed

lib/liquid/locales/en.yml

@@ -3,6 +3,7 @@
     syntax:
       tag_unexpected_args: "Syntax Error in '%{tag}' - Valid syntax: %{tag}"
       assign: "Syntax Error in 'assign' - Valid syntax: assign [var] = [source]"
+      local: "Syntax Error in 'local' - Valid syntax: local [var] = [source]"
       capture: "Syntax Error in 'capture' - Valid syntax: capture [var]"
       case: "Syntax Error in 'case' - Valid syntax: case [condition]"
       case_invalid_when: "Syntax Error in tag 'case' - Valid when condition: {% when [condition] [or condition2...] %}"

lib/liquid/tag.rb

@@ -5,8 +5,8 @@ module Liquid
     include ParserSwitching
 
     class << self
-      def parse(tag_name, markup, tokenizer, options)
-        tag = new(tag_name, markup, options)
+      def parse(tag_name, markup, tokenizer, parse_context)
+        tag = new(tag_name, markup, parse_context)
         tag.parse(tokenizer)
         tag
       end

lib/liquid/tags/assign.rb

@@ -10,6 +10,10 @@ module Liquid
   class Assign < Tag
     Syntax = /(#{VariableSignature}+)\s*=\s*(.*)\s*/om
 
+    def self.syntax_error_translation_key
+      "errors.syntax.assign".freeze
+    end
+
     attr_reader :to, :from
 
     def initialize(tag_name, markup, options)
@@ -18,7 +22,7 @@ module Liquid
         @to = $1
         @from = Variable.new($2, options)
       else
-        raise SyntaxError.new options[:locale].t("errors.syntax.assign".freeze)
+        raise SyntaxError.new(options[:locale].t(self.class.syntax_error_translation_key))
       end
     end
 

lib/liquid/tags/echo.rb

@@ -0,0 +1,24 @@
+module Liquid
+  # Echo outputs an expression
+  #
+  #   {% echo monkey %}
+  #   {% echo user.name %}
+  #
+  # This is identical to variable output syntax, like {{ foo }}, but works
+  # inside {% liquid %} tags. The full syntax is supported, including filters:
+  #
+  #   {% echo user | link %}
+  #
+  class Echo < Tag
+    def initialize(tag_name, markup, parse_context)
+      super
+      @variable = Variable.new(markup, parse_context)
+    end
+
+    def render(context)
+      @variable.render(context)
+    end
+  end
+
+  Template.register_tag('echo'.freeze, Echo)
+end

lib/liquid/tags/local.rb

@@ -0,0 +1,30 @@
+require_relative 'assign'
+
+module Liquid
+  # Local sets a variable in the current scope.
+  #
+  #   {% local foo = 'monkey' %}
+  #
+  # You can then use the variable later in the scope.
+  #
+  # {% if true %}
+  #   {% local foo = 'monkey' %}
+  #   {{ foo }} outputs monkey
+  # {% endif %}
+  # {{ foo }} outputs nothing
+  #
+  class Local < Assign
+    def self.syntax_error_translation_key
+      "errors.syntax.local".freeze
+    end
+
+    def render(context)
+      val = @from.render(context)
+      context[@to] = val
+      context.resource_limits.assign_score += assign_score_of(val)
+      ''.freeze
+    end
+  end
+
+  Template.register_tag('local'.freeze, Local)
+end

lib/liquid/tokenizer.rb

@@ -1,25 +1,31 @@
 module Liquid
   class Tokenizer
-    attr_reader :line_number
+    attr_reader :line_number, :for_liquid_tag
 
-    def initialize(source, line_numbers = false)
+    def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
       @source = source
-      @line_number = line_numbers ? 1 : nil
+      @line_number = line_number || (line_numbers ? 1 : nil)
+      @for_liquid_tag = for_liquid_tag
       @tokens = tokenize
     end
 
     def shift
-      token = @tokens.shift
-      @line_number += token.count("\n") if @line_number && token
+      token = @tokens.shift or return
+
+      if @line_number
+        @line_number += @for_liquid_tag ? 1 : token.count("\n")
+      end
+
       token
     end
 
     private
 
     def tokenize
-      @source = @source.source if @source.respond_to?(:source)
       return [] if @source.to_s.empty?
 
+      return @source.split("\n") if @for_liquid_tag
+
       tokens = @source.split(TemplateParser)
 
       # removes the rogue empty element at the beginning of the array

test/integration/tags/echo_test.rb

@@ -0,0 +1,11 @@
+require 'test_helper'
+
+class EchoTest < Minitest::Test
+  include Liquid
+
+  def test_echo_outputs_its_input
+    assert_template_result('BAR', <<~LIQUID, { 'variable-name' => 'bar' })
+      {%- echo variable-name | upcase -%}
+    LIQUID
+  end
+end

test/integration/tags/liquid_tag_test.rb

@@ -0,0 +1,98 @@
+require 'test_helper'
+
+class LiquidTagTest < Minitest::Test
+  include Liquid
+
+  def test_liquid_tag
+    assert_template_result('1 2 3', <<~LIQUID, 'array' => [1, 2, 3])
+    {%- liquid
+      echo array | join: " "
+    -%}
+    LIQUID
+
+    assert_template_result('1 2 3', <<~LIQUID, 'array' => [1, 2, 3])
+    {%- liquid
+      for value in array
+        echo value
+        unless forloop.last
+          echo " "
+        endunless
+      endfor
+    -%}
+    LIQUID
+
+    assert_template_result('4 8 12', <<~LIQUID, 'array' => [1, 2, 3])
+    {%- liquid
+      for value in array
+        local double_value = value | times: 2
+        echo double_value | times: 2
+        unless forloop.last
+          echo " "
+        endunless
+      endfor
+
+      echo double_value
+    -%}
+    LIQUID
+
+    assert_template_result('abc', <<~LIQUID)
+    {%- liquid echo "a" -%}
+    b
+    {%- liquid echo "c" -%}
+    LIQUID
+  end
+
+  def test_liquid_tag_errors
+    assert_match_syntax_error("syntax error (line 1): Unknown tag 'error'", <<~LIQUID)
+    {%- liquid error no such tag -%}
+    LIQUID
+
+    assert_match_syntax_error("syntax error (line 7): Unknown tag 'error'", <<~LIQUID)
+    {{ test }}
+
+    {%-
+    liquid
+      for value in array
+
+        error no such tag
+      endfor
+    -%}
+    LIQUID
+
+    assert_match_syntax_error("syntax error (line 2): Unknown tag '!!! the guards are vigilant'", <<~LIQUID)
+    {%- liquid
+      !!! the guards are vigilant
+    -%}
+    LIQUID
+
+    assert_match_syntax_error("syntax error (line 4): 'for' tag was never closed", <<~LIQUID)
+    {%- liquid
+      for value in array
+        echo 'forgot to close the for tag'
+    -%}
+    LIQUID
+  end
+
+  def test_cannot_open_blocks_living_past_a_liquid_tag
+    assert_match_syntax_error("syntax error (line 3): 'if' tag was never closed", <<~LIQUID)
+    {%- liquid
+      if true
+    -%}
+    {%- endif -%}
+    LIQUID
+  end
+
+  def test_quirk_can_close_blocks_created_before_a_liquid_tag
+    assert_template_result("42", <<~LIQUID)
+    {%- if true -%}
+    42
+    {%- liquid endif -%}
+    LIQUID
+  end
+
+  def test_liquid_tag_in_raw
+    assert_template_result("{% liquid echo 'test' %}\n", <<~LIQUID)
+    {% raw %}{% liquid echo 'test' %}{% endraw %}
+    LIQUID
+  end
+end

test/integration/tags/local_test.rb

@@ -0,0 +1,15 @@
+require 'test_helper'
+
+class LocalTest < Minitest::Test
+  include Liquid
+
+  def test_local_is_scope_aware
+    assert_template_result('value', <<~LIQUID)
+      {%- if true -%}
+        {%- local variable-name = 'value' -%}
+        {{- variable-name -}}
+      {%- endif -%}
+      {{- variable-name -}}
+    LIQUID
+  end
+end

test/test_helper.rb

@@ -37,18 +37,18 @@ module Minitest
     include Liquid
 
     def assert_template_result(expected, template, assigns = {}, message = nil)
-      assert_equal expected, Template.parse(template).render!(assigns), message
+      assert_equal expected, Template.parse(template, line_numbers: true).render!(assigns), message
     end
 
     def assert_template_result_matches(expected, template, assigns = {}, message = nil)
       return assert_template_result(expected, template, assigns, message) unless expected.is_a? Regexp
 
-      assert_match expected, Template.parse(template).render!(assigns), message
+      assert_match expected, Template.parse(template, line_numbers: true).render!(assigns), message
     end
 
     def assert_match_syntax_error(match, template, assigns = {})
       exception = assert_raises(Liquid::SyntaxError) do
-        Template.parse(template).render(assigns)
+        Template.parse(template, line_numbers: true).render(assigns)
       end
       assert_match match, exception.message
     end