From a07e382617df94177b38ac6adef3f428f9d12a6f Mon Sep 17 00:00:00 2001 From: Justin Li Date: Wed, 29 Oct 2014 11:28:41 -0400 Subject: [PATCH 1/3] Use a single token for identifiers --- lib/liquid/lexer.rb | 2 +- lib/liquid/parser.rb | 7 ------- test/unit/lexer_unit_test.rb | 2 +- test/unit/variable_unit_test.rb | 9 +++++++++ 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/liquid/lexer.rb b/lib/liquid/lexer.rb index cb8e591..d812fcc 100644 --- a/lib/liquid/lexer.rb +++ b/lib/liquid/lexer.rb @@ -13,7 +13,7 @@ module Liquid '?'.freeze => :question, '-'.freeze => :dash } - IDENTIFIER = /\w+/ + IDENTIFIER = /[\w-]+\??/ SINGLE_STRING_LITERAL = /'[^\']*'/ DOUBLE_STRING_LITERAL = /"[^\"]*"/ NUMBER_LITERAL = /-?\d+(\.\d+)?/ diff --git a/lib/liquid/parser.rb b/lib/liquid/parser.rb index ecf3634..d2006f3 100644 --- a/lib/liquid/parser.rb +++ b/lib/liquid/parser.rb @@ -75,13 +75,6 @@ module Liquid def variable_signature str = consume(:id) - while consume?(:dash) - str << "-".freeze - str << consume(:id) - end - if consume?(:question) - str << "?".freeze - end if look(:open_square) str << consume str << expression diff --git a/test/unit/lexer_unit_test.rb b/test/unit/lexer_unit_test.rb index ef341a1..cf268d1 100644 --- a/test/unit/lexer_unit_test.rb +++ b/test/unit/lexer_unit_test.rb @@ -32,7 +32,7 @@ class LexerUnitTest < Minitest::Test def test_fancy_identifiers tokens = Lexer.new('hi five?').tokenize - assert_equal [[:id,'hi'], [:id, 'five'], [:question, '?'], [:end_of_string]], tokens + assert_equal [[:id,'hi'], [:id, 'five?'], [:end_of_string]], tokens end def test_whitespace diff --git a/test/unit/variable_unit_test.rb b/test/unit/variable_unit_test.rb index 13d2cc4..9e170d4 100644 --- a/test/unit/variable_unit_test.rb +++ b/test/unit/variable_unit_test.rb @@ -102,6 +102,15 @@ class VariableUnitTest < Minitest::Test assert_equal 1000.01, var.name end + def test_dashes + assert_equal VariableLookup.new('foo-bar'), Variable.new('foo-bar').name + assert_equal VariableLookup.new('foo-bar-2'), Variable.new('foo-bar-2').name + + with_error_mode :strict do + assert_raises(Liquid::SyntaxError) { Variable.new('foo - bar') } + end + end + def test_string_with_special_chars var = Variable.new(%| 'hello! $!@.;"ddasd" ' |) assert_equal 'hello! $!@.;"ddasd" ', var.name From dd5ee81089c9352f248d48f2114f34402b271a8e Mon Sep 17 00:00:00 2001 From: Justin Li Date: Wed, 29 Oct 2014 12:08:00 -0400 Subject: [PATCH 2/3] Disallow number and dash identifier prefixes --- lib/liquid/lexer.rb | 2 +- lib/liquid/variable.rb | 8 -------- test/unit/lexer_unit_test.rb | 5 ++++- test/unit/variable_unit_test.rb | 2 ++ 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/liquid/lexer.rb b/lib/liquid/lexer.rb index d812fcc..7ab831c 100644 --- a/lib/liquid/lexer.rb +++ b/lib/liquid/lexer.rb @@ -13,7 +13,7 @@ module Liquid '?'.freeze => :question, '-'.freeze => :dash } - IDENTIFIER = /[\w-]+\??/ + IDENTIFIER = /[a-zA-Z_]+[\w-]*\??/ SINGLE_STRING_LITERAL = /'[^\']*'/ DOUBLE_STRING_LITERAL = /"[^\"]*"/ NUMBER_LITERAL = /-?\d+(\.\d+)?/ diff --git a/lib/liquid/variable.rb b/lib/liquid/variable.rb index a93fed7..92b0537 100644 --- a/lib/liquid/variable.rb +++ b/lib/liquid/variable.rb @@ -12,7 +12,6 @@ module Liquid # class Variable FilterParser = /(?:\s+|#{QuotedFragment}|#{ArgumentSeparator})+/o - EasyParse = /\A *(\w+(?:\.\w+)*) *\z/ attr_accessor :filters, :name, :warnings attr_accessor :line_number include ParserSwitching @@ -53,13 +52,6 @@ module Liquid end def strict_parse(markup) - # Very simple valid cases - if markup =~ EasyParse - @name = Expression.parse($1) - @filters = [] - return - end - @filters = [] p = Parser.new(markup) diff --git a/test/unit/lexer_unit_test.rb b/test/unit/lexer_unit_test.rb index cf268d1..c331380 100644 --- a/test/unit/lexer_unit_test.rb +++ b/test/unit/lexer_unit_test.rb @@ -32,7 +32,10 @@ class LexerUnitTest < Minitest::Test def test_fancy_identifiers tokens = Lexer.new('hi five?').tokenize - assert_equal [[:id,'hi'], [:id, 'five?'], [:end_of_string]], tokens + assert_equal [[:id, 'hi'], [:id, 'five?'], [:end_of_string]], tokens + + tokens = Lexer.new('2foo').tokenize + assert_equal [[:number, '2'], [:id, 'foo'], [:end_of_string]], tokens end def test_whitespace diff --git a/test/unit/variable_unit_test.rb b/test/unit/variable_unit_test.rb index 9e170d4..084b846 100644 --- a/test/unit/variable_unit_test.rb +++ b/test/unit/variable_unit_test.rb @@ -108,6 +108,8 @@ class VariableUnitTest < Minitest::Test with_error_mode :strict do assert_raises(Liquid::SyntaxError) { Variable.new('foo - bar') } + assert_raises(Liquid::SyntaxError) { Variable.new('-foo') } + assert_raises(Liquid::SyntaxError) { Variable.new('2foo') } end end From 81770f094dfeca868cdef44b6681ee51b6860a47 Mon Sep 17 00:00:00 2001 From: Justin Li Date: Wed, 29 Oct 2014 13:39:43 -0400 Subject: [PATCH 3/3] Remove unnecessary + --- lib/liquid/lexer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/liquid/lexer.rb b/lib/liquid/lexer.rb index 7ab831c..4b724b1 100644 --- a/lib/liquid/lexer.rb +++ b/lib/liquid/lexer.rb @@ -13,7 +13,7 @@ module Liquid '?'.freeze => :question, '-'.freeze => :dash } - IDENTIFIER = /[a-zA-Z_]+[\w-]*\??/ + IDENTIFIER = /[a-zA-Z_][\w-]*\??/ SINGLE_STRING_LITERAL = /'[^\']*'/ DOUBLE_STRING_LITERAL = /"[^\"]*"/ NUMBER_LITERAL = /-?\d+(\.\d+)?/