Merge pull request #478 from Shopify/numbers-in-identifiers

Use a single token for identifiers

lib/liquid/lexer.rb

@@ -13,7 +13,7 @@ module Liquid
       '?'.freeze => :question,
       '-'.freeze => :dash
     }
-    IDENTIFIER = /\w+/
+    IDENTIFIER = /[a-zA-Z_][\w-]*\??/
     SINGLE_STRING_LITERAL = /'[^\']*'/
     DOUBLE_STRING_LITERAL = /"[^\"]*"/
     NUMBER_LITERAL = /-?\d+(\.\d+)?/

lib/liquid/parser.rb

@@ -75,13 +75,6 @@ module Liquid
 
     def variable_signature
       str = consume(:id)
-      while consume?(:dash)
-        str << "-".freeze
-        str << consume(:id)
-      end
-      if consume?(:question)
-        str << "?".freeze
-      end
       if look(:open_square)
         str << consume
         str << expression

lib/liquid/variable.rb

@@ -12,7 +12,6 @@ module Liquid
   #
   class Variable
     FilterParser = /(?:\s+|#{QuotedFragment}|#{ArgumentSeparator})+/o
-    EasyParse = /\A *(\w+(?:\.\w+)*) *\z/
     attr_accessor :filters, :name, :warnings
     attr_accessor :line_number
     include ParserSwitching
@@ -53,13 +52,6 @@ module Liquid
     end
 
     def strict_parse(markup)
-      # Very simple valid cases
-      if markup =~ EasyParse
-        @name = Expression.parse($1)
-        @filters = []
-        return
-      end
-
       @filters = []
       p = Parser.new(markup)
 

test/unit/lexer_unit_test.rb

@@ -32,7 +32,10 @@ class LexerUnitTest < Minitest::Test
 
   def test_fancy_identifiers
     tokens = Lexer.new('hi five?').tokenize
-    assert_equal [[:id,'hi'], [:id, 'five'], [:question, '?'], [:end_of_string]], tokens
+    assert_equal [[:id, 'hi'], [:id, 'five?'], [:end_of_string]], tokens
+
+    tokens = Lexer.new('2foo').tokenize
+    assert_equal [[:number, '2'], [:id, 'foo'], [:end_of_string]], tokens
   end
 
   def test_whitespace

test/unit/variable_unit_test.rb

@@ -102,6 +102,17 @@ class VariableUnitTest < Minitest::Test
     assert_equal 1000.01, var.name
   end
 
+  def test_dashes
+    assert_equal VariableLookup.new('foo-bar'), Variable.new('foo-bar').name
+    assert_equal VariableLookup.new('foo-bar-2'), Variable.new('foo-bar-2').name
+
+    with_error_mode :strict do
+      assert_raises(Liquid::SyntaxError) { Variable.new('foo - bar') }
+      assert_raises(Liquid::SyntaxError) { Variable.new('-foo') }
+      assert_raises(Liquid::SyntaxError) { Variable.new('2foo') }
+    end
+  end
+
   def test_string_with_special_chars
     var = Variable.new(%| 'hello! $!@.;"ddasd" ' |)
     assert_equal 'hello! $!@.;"ddasd" ', var.name