From 8b98f92c7f1883989f9e54739c551872a503dde3 Mon Sep 17 00:00:00 2001 From: Gaurav Chande Date: Wed, 3 Jun 2015 19:40:49 +0000 Subject: [PATCH 1/4] Extract tokenize logic from Template to a RubyTokenizer --- lib/liquid.rb | 1 + lib/liquid/template.rb | 22 +----------------- lib/liquid/tokenizer.rb | 38 ++++++++++++++++++++++++++++++++ test/unit/tokenizer_unit_test.rb | 11 ++++----- 4 files changed, 46 insertions(+), 26 deletions(-) create mode 100644 lib/liquid/tokenizer.rb diff --git a/lib/liquid.rb b/lib/liquid.rb index 9724aac..14afa75 100644 --- a/lib/liquid.rb +++ b/lib/liquid.rb @@ -68,6 +68,7 @@ require 'liquid/template' require 'liquid/standardfilters' require 'liquid/condition' require 'liquid/utils' +require 'liquid/tokenizer' require 'liquid/token' # Load all the tags of the standard library diff --git a/lib/liquid/template.rb b/lib/liquid/template.rb index a6e36b1..05b4114 100644 --- a/lib/liquid/template.rb +++ b/lib/liquid/template.rb @@ -228,28 +228,8 @@ module Liquid private - # Uses the Liquid::TemplateParser regexp to tokenize the passed source def tokenize(source) - source = source.source if source.respond_to?(:source) - return [] if source.to_s.empty? - - tokens = calculate_line_numbers(source.split(TemplateParser)) - - # removes the rogue empty element at the beginning of the array - tokens.shift if tokens[0] && tokens[0].empty? - - tokens - end - - def calculate_line_numbers(raw_tokens) - return raw_tokens unless @line_numbers - - current_line = 1 - raw_tokens.map do |token| - Token.new(token, current_line).tap do - current_line += token.count("\n") - end - end + Tokenizer.new(source, @line_numbers) end def with_profiling diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb new file mode 100644 index 0000000..9196d2a --- /dev/null +++ b/lib/liquid/tokenizer.rb @@ -0,0 +1,38 @@ +module Liquid + class Tokenizer + attr_reader :tokens + + def initialize(source, line_numbers = false) + @source, @line_numbers = source, line_numbers + @tokens = tokenize + end + + def shift + @tokens.shift + end + + private + + def tokenize + @source = @source.source if @source.respond_to?(:source) + return [] if @source.to_s.empty? + + tokens = @source.split(TemplateParser) + tokens = @line_numbers ? calculate_line_numbers(tokens) : tokens + + # removes the rogue empty element at the beginning of the array + tokens.shift if tokens[0] && tokens[0].empty? + + tokens + end + + def calculate_line_numbers(tokens) + current_line = 1 + tokens.map do |token| + Token.new(token, current_line).tap do + current_line += token.count("\n") + end + end + end + end +end diff --git a/test/unit/tokenizer_unit_test.rb b/test/unit/tokenizer_unit_test.rb index f0ae7c9..71b6205 100644 --- a/test/unit/tokenizer_unit_test.rb +++ b/test/unit/tokenizer_unit_test.rb @@ -24,15 +24,16 @@ class TokenizerTest < Minitest::Test def test_calculate_line_numbers_per_token_with_profiling template = Liquid::Template.parse("", :profile => true) - assert_equal [1], template.send(:tokenize, "{{funk}}").map(&:line_number) - assert_equal [1, 1, 1], template.send(:tokenize, " {{funk}} ").map(&:line_number) - assert_equal [1, 2, 2], template.send(:tokenize, "\n{{funk}}\n").map(&:line_number) - assert_equal [1, 1, 3], template.send(:tokenize, " {{\n funk \n}} ").map(&:line_number) + assert_equal [1], template.send(:tokenize, "{{funk}}").tokens.map(&:line_number) + assert_equal [1, 1, 1], template.send(:tokenize, " {{funk}} ").tokens.map(&:line_number) + assert_equal [1, 2, 2], template.send(:tokenize, "\n{{funk}}\n").tokens.map(&:line_number) + assert_equal [1, 1, 3], template.send(:tokenize, " {{\n funk \n}} ").tokens.map(&:line_number) end private def tokenize(source) - Liquid::Template.new.send(:tokenize, source) + tokenizer = Liquid::Tokenizer.new(source) + tokenizer.tokens end end From 3a907a4db7e144dd4957d925993f357b109d068d Mon Sep 17 00:00:00 2001 From: Gaurav Chande Date: Wed, 3 Jun 2015 19:42:16 +0000 Subject: [PATCH 2/4] Move DEFAULT_OPTIONS related logic to Document --- lib/liquid/document.rb | 6 +++++- lib/liquid/template.rb | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/liquid/document.rb b/lib/liquid/document.rb index 7ecced6..c964ae8 100644 --- a/lib/liquid/document.rb +++ b/lib/liquid/document.rb @@ -1,8 +1,12 @@ module Liquid class Document < BlockBody + DEFAULT_OPTIONS = { + locale: I18n.new + } + def self.parse(tokens, options) doc = new - doc.parse(tokens, options) + doc.parse(tokens, DEFAULT_OPTIONS.merge(options)) doc end diff --git a/lib/liquid/template.rb b/lib/liquid/template.rb index 05b4114..7b2897d 100644 --- a/lib/liquid/template.rb +++ b/lib/liquid/template.rb @@ -13,10 +13,6 @@ module Liquid # template.render('user_name' => 'bob') # class Template - DEFAULT_OPTIONS = { - locale: I18n.new - } - attr_accessor :root attr_reader :resource_limits @@ -120,7 +116,7 @@ module Liquid @options = options @profiling = options[:profile] @line_numbers = options[:line_numbers] || @profiling - @root = Document.parse(tokenize(source), DEFAULT_OPTIONS.merge(options)) + @root = Document.parse(tokenize(source), options) @warnings = nil self end From 79d7dd06df99c709ac2f97f4d38472d36395a34d Mon Sep 17 00:00:00 2001 From: Gaurav Chande Date: Wed, 3 Jun 2015 19:44:52 +0000 Subject: [PATCH 3/4] Extract tag fetching into a method (which can be overriden then) --- lib/liquid/block_body.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/liquid/block_body.rb b/lib/liquid/block_body.rb index 66eaffa..be5b65e 100644 --- a/lib/liquid/block_body.rb +++ b/lib/liquid/block_body.rb @@ -22,7 +22,7 @@ module Liquid tag_name = $1 markup = $2 # fetch the tag from registered blocks - if tag = Template.tags[tag_name] + if tag = registered_tags[tag_name] markup = token.child(markup) if token.is_a?(Token) new_tag = tag.parse(tag_name, markup, tokens, options) new_tag.line_number = token.line_number if token.is_a?(Token) @@ -127,5 +127,9 @@ module Liquid def raise_missing_variable_terminator(token, options) raise SyntaxError.new(options[:locale].t("errors.syntax.variable_termination".freeze, token: token, tag_end: VariableEnd.inspect)) end + + def registered_tags + Template.tags + end end end From fc8e6c8d3a2a2a550e7baa16fd25b76689ef697e Mon Sep 17 00:00:00 2001 From: Gaurav Chande Date: Thu, 4 Jun 2015 05:48:39 +0000 Subject: [PATCH 4/4] Change Tokenizer test to fetch tokens instead of exposing ivar --- lib/liquid/tokenizer.rb | 2 -- test/unit/tokenizer_unit_test.rb | 20 +++++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb index 9196d2a..51347ed 100644 --- a/lib/liquid/tokenizer.rb +++ b/lib/liquid/tokenizer.rb @@ -1,7 +1,5 @@ module Liquid class Tokenizer - attr_reader :tokens - def initialize(source, line_numbers = false) @source, @line_numbers = source, line_numbers @tokens = tokenize diff --git a/test/unit/tokenizer_unit_test.rb b/test/unit/tokenizer_unit_test.rb index 71b6205..e9cabfa 100644 --- a/test/unit/tokenizer_unit_test.rb +++ b/test/unit/tokenizer_unit_test.rb @@ -22,18 +22,20 @@ class TokenizerTest < Minitest::Test end def test_calculate_line_numbers_per_token_with_profiling - template = Liquid::Template.parse("", :profile => true) - - assert_equal [1], template.send(:tokenize, "{{funk}}").tokens.map(&:line_number) - assert_equal [1, 1, 1], template.send(:tokenize, " {{funk}} ").tokens.map(&:line_number) - assert_equal [1, 2, 2], template.send(:tokenize, "\n{{funk}}\n").tokens.map(&:line_number) - assert_equal [1, 1, 3], template.send(:tokenize, " {{\n funk \n}} ").tokens.map(&:line_number) + assert_equal [1], tokenize("{{funk}}", true).map(&:line_number) + assert_equal [1, 1, 1], tokenize(" {{funk}} ", true).map(&:line_number) + assert_equal [1, 2, 2], tokenize("\n{{funk}}\n", true).map(&:line_number) + assert_equal [1, 1, 3], tokenize(" {{\n funk \n}} ", true).map(&:line_number) end private - def tokenize(source) - tokenizer = Liquid::Tokenizer.new(source) - tokenizer.tokens + def tokenize(source, line_numbers = false) + tokenizer = Liquid::Tokenizer.new(source, line_numbers) + tokens = [] + while t = tokenizer.shift + tokens << t + end + tokens end end