Merge pull request #590 from Shopify/allow-template-tags

Local Tags

lib/liquid.rb

@@ -68,6 +68,7 @@ require 'liquid/template'
 require 'liquid/standardfilters'
 require 'liquid/condition'
 require 'liquid/utils'
+require 'liquid/tokenizer'
 require 'liquid/token'
 
 # Load all the tags of the standard library

lib/liquid/block_body.rb

@@ -22,7 +22,7 @@ module Liquid
                 tag_name = $1
                 markup = $2
                 # fetch the tag from registered blocks
-                if tag = Template.tags[tag_name]
+                if tag = registered_tags[tag_name]
                   markup = token.child(markup) if token.is_a?(Token)
                   new_tag = tag.parse(tag_name, markup, tokens, options)
                   new_tag.line_number = token.line_number if token.is_a?(Token)
@@ -127,5 +127,9 @@ module Liquid
     def raise_missing_variable_terminator(token, options)
       raise SyntaxError.new(options[:locale].t("errors.syntax.variable_termination".freeze, token: token, tag_end: VariableEnd.inspect))
     end
+
+    def registered_tags
+      Template.tags
+    end
   end
 end

lib/liquid/document.rb

@@ -1,8 +1,12 @@
 module Liquid
   class Document < BlockBody
+    DEFAULT_OPTIONS = {
+      locale: I18n.new
+    }
+
     def self.parse(tokens, options)
       doc = new
-      doc.parse(tokens, options)
+      doc.parse(tokens, DEFAULT_OPTIONS.merge(options))
       doc
     end
 

lib/liquid/template.rb

@@ -13,10 +13,6 @@ module Liquid
   #   template.render('user_name' => 'bob')
   #
   class Template
-    DEFAULT_OPTIONS = {
-      locale: I18n.new
-    }
-
     attr_accessor :root
     attr_reader :resource_limits
 
@@ -120,7 +116,7 @@ module Liquid
       @options = options
       @profiling = options[:profile]
       @line_numbers = options[:line_numbers] || @profiling
-      @root = Document.parse(tokenize(source), DEFAULT_OPTIONS.merge(options))
+      @root = Document.parse(tokenize(source), options)
       @warnings = nil
       self
     end
@@ -228,28 +224,8 @@ module Liquid
 
     private
 
-    # Uses the <tt>Liquid::TemplateParser</tt> regexp to tokenize the passed source
     def tokenize(source)
-      source = source.source if source.respond_to?(:source)
-      return [] if source.to_s.empty?
-
-      tokens = calculate_line_numbers(source.split(TemplateParser))
-
-      # removes the rogue empty element at the beginning of the array
-      tokens.shift if tokens[0] && tokens[0].empty?
-
-      tokens
-    end
-
-    def calculate_line_numbers(raw_tokens)
-      return raw_tokens unless @line_numbers
-
-      current_line = 1
-      raw_tokens.map do |token|
-        Token.new(token, current_line).tap do
-          current_line += token.count("\n")
-        end
-      end
+      Tokenizer.new(source, @line_numbers)
     end
 
     def with_profiling

lib/liquid/tokenizer.rb

@@ -0,0 +1,36 @@
+module Liquid
+  class Tokenizer
+    def initialize(source, line_numbers = false)
+      @source, @line_numbers = source, line_numbers
+      @tokens = tokenize
+    end
+
+    def shift
+      @tokens.shift
+    end
+
+    private
+
+    def tokenize
+      @source = @source.source if @source.respond_to?(:source)
+      return [] if @source.to_s.empty?
+
+      tokens = @source.split(TemplateParser)
+      tokens = @line_numbers ? calculate_line_numbers(tokens) : tokens
+
+      # removes the rogue empty element at the beginning of the array
+      tokens.shift if tokens[0] && tokens[0].empty?
+
+      tokens
+    end
+
+    def calculate_line_numbers(tokens)
+      current_line = 1
+      tokens.map do |token|
+        Token.new(token, current_line).tap do
+          current_line += token.count("\n")
+        end
+      end
+    end
+  end
+end

test/unit/tokenizer_unit_test.rb

@@ -22,17 +22,20 @@ class TokenizerTest < Minitest::Test
   end
 
   def test_calculate_line_numbers_per_token_with_profiling
-    template = Liquid::Template.parse("", :profile => true)
-
-    assert_equal [1],       template.send(:tokenize, "{{funk}}").map(&:line_number)
-    assert_equal [1, 1, 1], template.send(:tokenize, " {{funk}} ").map(&:line_number)
-    assert_equal [1, 2, 2], template.send(:tokenize, "\n{{funk}}\n").map(&:line_number)
-    assert_equal [1, 1, 3], template.send(:tokenize, " {{\n funk \n}} ").map(&:line_number)
+    assert_equal [1],       tokenize("{{funk}}", true).map(&:line_number)
+    assert_equal [1, 1, 1], tokenize(" {{funk}} ", true).map(&:line_number)
+    assert_equal [1, 2, 2], tokenize("\n{{funk}}\n", true).map(&:line_number)
+    assert_equal [1, 1, 3], tokenize(" {{\n funk \n}} ", true).map(&:line_number)
   end
 
   private
 
-  def tokenize(source)
-    Liquid::Template.new.send(:tokenize, source)
+  def tokenize(source, line_numbers = false)
+    tokenizer = Liquid::Tokenizer.new(source, line_numbers)
+    tokens = []
+    while t = tokenizer.shift
+      tokens << t
+    end
+    tokens
   end
 end