kemko/liquid
1
0
Fork 0
mirror of https://github.com/kemko/liquid.git synced 2026-01-01 15:55:40 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

Build the tokenizer through the parse context for liquid-c (#1386)

Browse Source
This commit is contained in:
Dylan Thacker-Smith
2021-01-07 14:51:41 -05:00
committed by GitHub
parent 42b6c07cd0
commit 260c863e23
4 changed files with 18 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
test/unit/tokenizer_unit_test.rb
View File

@@ -32,21 +32,26 @@ class TokenizerTest < Minitest::Test
private
def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)
parse_context.new_tokenizer(source, start_line_number: start_line_number)
end
def tokenize(source)
tokenizer = Liquid::Tokenizer.new(source)
tokenizer = new_tokenizer(source)
tokens = []
while (t = tokenizer.shift)
# shift is private in Liquid::C::Tokenizer, since it is only for unit testing
while (t = tokenizer.send(:shift))
tokens << t
end
tokens
end
def tokenize_line_numbers(source)
tokenizer = Liquid::Tokenizer.new(source, true)
tokenizer = new_tokenizer(source, start_line_number: 1)
line_numbers = []
loop do
line_number = tokenizer.line_number
if tokenizer.shift
if tokenizer.send(:shift)
line_numbers << line_number
else
break