From 260c863e23c5e69dac9de91e1fc176b479b44769 Mon Sep 17 00:00:00 2001
From: Dylan Thacker-Smith <dylan.smith@shopify.com>
Date: Thu, 7 Jan 2021 14:51:41 -0500
Subject: [PATCH] Build the tokenizer through the parse context for liquid-c
 (#1386)

---
 lib/liquid/block_body.rb         |  4 +++-
 lib/liquid/parse_context.rb      |  4 ++++
 lib/liquid/template.rb           |  7 ++-----
 test/unit/tokenizer_unit_test.rb | 13 +++++++++----
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/lib/liquid/block_body.rb b/lib/liquid/block_body.rb
index af62a14..76ab0a8 100644
--- a/lib/liquid/block_body.rb
+++ b/lib/liquid/block_body.rb
@@ -99,7 +99,9 @@ module Liquid
     end
 
     private def parse_liquid_tag(markup, parse_context)
-      liquid_tag_tokenizer = Tokenizer.new(markup, line_number: parse_context.line_number, for_liquid_tag: true)
+      liquid_tag_tokenizer = parse_context.new_tokenizer(
+        markup, start_line_number: parse_context.line_number, for_liquid_tag: true
+      )
       parse_for_liquid_tag(liquid_tag_tokenizer, parse_context) do |end_tag_name, _end_tag_markup|
         if end_tag_name
           BlockBody.unknown_tag_in_liquid_tag(end_tag_name, parse_context)
diff --git a/lib/liquid/parse_context.rb b/lib/liquid/parse_context.rb
index f8fb9b0..87570ad 100644
--- a/lib/liquid/parse_context.rb
+++ b/lib/liquid/parse_context.rb
@@ -23,6 +23,10 @@ module Liquid
       Liquid::BlockBody.new
     end
 
+    def new_tokenizer(markup, start_line_number: nil, for_liquid_tag: false)
+      Tokenizer.new(markup, line_number: start_line_number, for_liquid_tag: for_liquid_tag)
+    end
+
     def parse_expression(markup)
       Expression.parse(markup)
     end
diff --git a/lib/liquid/template.rb b/lib/liquid/template.rb
index adcf6bd..cb8919a 100644
--- a/lib/liquid/template.rb
+++ b/lib/liquid/template.rb
@@ -107,7 +107,8 @@ module Liquid
     # Returns self for easy chaining
     def parse(source, options = {})
       parse_context = configure_options(options)
-      @root         = Document.parse(tokenize(source), parse_context)
+      tokenizer     = parse_context.new_tokenizer(source, start_line_number: @line_numbers && 1)
+      @root         = Document.parse(tokenizer, parse_context)
       self
     end
 
@@ -223,10 +224,6 @@ module Liquid
       parse_context
     end
 
-    def tokenize(source)
-      Tokenizer.new(source, @line_numbers)
-    end
-
     def apply_options_to_context(context, options)
       context.add_filters(options[:filters]) if options[:filters]
       context.global_filter      = options[:global_filter] if options[:global_filter]
diff --git a/test/unit/tokenizer_unit_test.rb b/test/unit/tokenizer_unit_test.rb
index e555d3a..6e85572 100644
--- a/test/unit/tokenizer_unit_test.rb
+++ b/test/unit/tokenizer_unit_test.rb
@@ -32,21 +32,26 @@ class TokenizerTest < Minitest::Test
 
   private
 
+  def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)
+    parse_context.new_tokenizer(source, start_line_number: start_line_number)
+  end
+
   def tokenize(source)
-    tokenizer = Liquid::Tokenizer.new(source)
+    tokenizer = new_tokenizer(source)
     tokens    = []
-    while (t = tokenizer.shift)
+    # shift is private in Liquid::C::Tokenizer, since it is only for unit testing
+    while (t = tokenizer.send(:shift))
       tokens << t
     end
     tokens
   end
 
   def tokenize_line_numbers(source)
-    tokenizer    = Liquid::Tokenizer.new(source, true)
+    tokenizer    = new_tokenizer(source, start_line_number: 1)
     line_numbers = []
     loop do
       line_number = tokenizer.line_number
-      if tokenizer.shift
+      if tokenizer.send(:shift)
         line_numbers << line_number
       else
         break