seperator -> separator

History.md

@@ -15,7 +15,6 @@
 * Remove support for `liquid_methods`
 
 ### Fixed
-* Fix test failure under certain timezones (#631) [Dylan Thacker-Smith]
 * Fix bug in uniq filter (#595) [Florian Weingarten]
 * Fix bug when "blank" and "empty" are used as variable names (#592) [Florian Weingarten]
 * Fix condition parse order in strict mode (#569) [Justin Li]
@@ -27,15 +26,7 @@
 * Disallow variable names in the strict parser that are not valid in the lax parser (#463) [Justin Li]
 * Fix BlockBody#warnings taking exponential time to compute (#486) [Justin Li]
 
-## 3.0.5 / 2015-07-23 / branch "3-0-stable"
-
-* Fix test failure under certain timezones [Dylan Thacker-Smith]
-
-## 3.0.4 / 2015-07-17
-
-* Fix chained access to multi-dimensional hashes [Florian Weingarten]
-
-## 3.0.3 / 2015-05-28
+## 3.0.3 / 2015-05-28 / branch "3-0-stable"
 
 * Fix condition parse order in strict mode (#569) [Justin Li]
 
@@ -83,15 +74,7 @@
 * Make map filter work on enumerable drops (#233) [Florian Weingarten]
 * Improved whitespace stripping for blank blocks, related to #216 [Florian Weingarten]
 
-## 2.6.3 / 2015-07-23 / branch "2-6-stable"
-
-* Fix test failure under certain timezones [Dylan Thacker-Smith]
-
-## 2.6.2 / 2015-01-23
-
-* Remove duplicate hash key [Parker Moore]
-
-## 2.6.1 / 2014-01-10
+## 2.6.1 / 2014-01-10 / branch "2-6-stable"
 
 Security fix, cherry-picked from master (4e14a65):
 * Don't call to_sym when creating conditions for security reasons (#273) [Bouke van der Bijl]

lib/liquid/lexer.rb

@@ -13,7 +13,7 @@ module Liquid
       '?'.freeze => :question,
       '-'.freeze => :dash
     }
-    IDENTIFIER = /[a-zA-Z_](?:[\w-]*\w)?\??/
+    IDENTIFIER = /[a-zA-Z_][\w-]*\??/
     SINGLE_STRING_LITERAL = /'[^\']*'/
     DOUBLE_STRING_LITERAL = /"[^\"]*"/
     NUMBER_LITERAL = /-?\d+(\.\d+)?/

lib/liquid/parser.rb

@@ -75,7 +75,7 @@ module Liquid
 
     def variable_signature
       str = consume(:id)
-      while look(:open_square)
+      if look(:open_square)
         str << consume
         str << expression
         str << consume(:close_square)

lib/liquid/parser_switching.rb

@@ -3,7 +3,7 @@ module Liquid
     def parse_with_selected_parser(markup)
       case parse_context.error_mode
       when :strict then strict_parse_with_error_context(markup)
-      when :lax    then lax_parse(markup)
+      when :lax, :lax_warn then lax_parse(markup)
       when :warn
         begin
           return strict_parse_with_error_context(markup)

lib/liquid/utils.rb

@@ -72,7 +72,7 @@ module Liquid
       when String
         Time.parse(obj)
       end
-    rescue ::ArgumentError
+    rescue ArgumentError
       nil
     end
   end

lib/liquid/variable.rb

@@ -10,7 +10,16 @@ module Liquid
   #   {{ user | link }}
   #
   class Variable
-    FilterParser = /(?:\s+|#{QuotedFragment}|#{ArgumentSeparator})+/o
+    capture_ignored_variable_prefix = /([\s,\|'"]+?)??/
+    capture_expression = /(#{QuotedFragment})/o
+    capture_ignored_filter_prefix = /([^\|]+?)??/
+    capture_filters = /(#{FilterSeparator}.*)/o
+    VariableSyntax = /\A\s*#{capture_ignored_variable_prefix}\s*#{capture_expression}\s*(?:#{capture_ignored_filter_prefix}\s*#{capture_filters})?\z/om
+
+    capture_lax_separator = /(['"\|]+?)/
+    capture_filter = /((?:\s+|#{QuotedFragment}|#{ArgumentSeparator})+)/o
+    FilterParser = /\s*(?:#{FilterSeparator}|#{capture_lax_separator})\s*#{capture_filter}/o
+
     attr_accessor :filters, :name, :line_number
     attr_reader :parse_context
     alias_method :options, :parse_context
@@ -35,16 +44,19 @@ module Liquid
 
     def lax_parse(markup)
       @filters = []
-      return unless markup =~ /(#{QuotedFragment})(.*)/om
+      return unless markup =~ VariableSyntax
 
-      name_markup = $1
-      filter_markup = $2
+      add_syntax_warning("variable prefixed with ignored characters: #{$1.inspect}") if $1
+      name_markup = $2
+      add_syntax_warning("variable filter separator prefixed with ignored characters: #{$3.inspect}") if $3
+      filters_markup = $4
       @name = Expression.parse(name_markup)
-      if filter_markup =~ /#{FilterSeparator}\s*(.*)/om
-        filters = $1.scan(FilterParser)
-        filters.each do |f|
-          next unless f =~ /\w+/
-          filtername = Regexp.last_match(0)
+      if filters_markup
+        filters_markup.scan(FilterParser) do |lax_sep, f|
+          add_syntax_warning("unterminated quote or multiple pipe characters used as a filter separator: #{lax_sep.inspect}") if lax_sep
+          next unless f =~ /\A\s*(\W+)??(\w+)/
+          add_syntax_warning("ignored characters before filter name: #{$1.inspect}") if $1
+          filtername = $2
           filterargs = f.scan(/(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o).flatten
           @filters << parse_filter_expressions(filtername, filterargs)
         end
@@ -81,6 +93,13 @@ module Liquid
 
     private
 
+    def add_syntax_warning(warning)
+      return unless parse_context.error_mode == :lax_warn
+      error = SyntaxError.new(warning)
+      error.line_number = parse_context.line_number
+      parse_context.warnings << error
+    end
+
     def parse_filter_expressions(filter_name, unparsed_args)
       filter_args = []
       keyword_args = {}

test/integration/output_test.rb

@@ -43,14 +43,6 @@ class OutputTest < Minitest::Test
     assert_equal expected, Template.parse(text).render!(@assigns)
   end
 
-  def test_variable_traversing_with_two_brackets
-    text = %({{ site.data.menu[include.menu][include.locale] }})
-    assert_equal "it works!", Template.parse(text).render!(
-      "site" => { "data" => { "menu" => { "foo" => { "bar" => "it works!" } } } },
-      "include" => { "menu" => "foo", "locale" => "bar" }
-    )
-  end
-
   def test_variable_traversing
     text = %( {{car.bmw}} {{car.gm}} {{car.bmw}} )
 

test/integration/standard_filter_test.rb

@@ -263,10 +263,8 @@ class StandardFiltersTest < Minitest::Test
 
     assert_equal '', @filters.date('', "%B")
 
-    with_timezone("UTC") do
-      assert_equal "07/05/2006", @filters.date(1152098955, "%m/%d/%Y")
-      assert_equal "07/05/2006", @filters.date("1152098955", "%m/%d/%Y")
-    end
+    assert_equal "07/05/2006", @filters.date(1152098955, "%m/%d/%Y")
+    assert_equal "07/05/2006", @filters.date("1152098955", "%m/%d/%Y")
   end
 
   def test_first_last
@@ -419,19 +417,4 @@ class StandardFiltersTest < Minitest::Test
   def test_cannot_access_private_methods
     assert_template_result('a', "{{ 'a' | to_number }}")
   end
-
-  def test_date_raises_nothing
-    assert_template_result('', "{{ '' | date: '%D' }}")
-    assert_template_result('abc', "{{ 'abc' | date: '%D' }}")
-  end
-
-  private
-
-  def with_timezone(tz)
-    old_tz = ENV['TZ']
-    ENV['TZ'] = tz
-    yield
-  ensure
-    ENV['TZ'] = old_tz
-  end
 end # StandardFiltersTest

test/integration/variable_test.rb

@@ -89,4 +89,32 @@ class VariableTest < Minitest::Test
   def test_multiline_variable
     assert_equal 'worked', Template.parse("{{\ntest\n}}").render!('test' => 'worked')
   end
+
+  def test_lax_warnings_for_variable_ignored_prefix
+    ignored_chars = %(|,'")
+    template = Liquid::Template.parse("{{ #{ignored_chars}test }}", error_mode: :lax_warn)
+    assert_equal "works", template.render!('test' => 'works')
+    assert_equal [Liquid::SyntaxError.new("variable prefixed with ignored characters: #{ignored_chars.inspect}")], template.warnings
+  end
+
+  def test_lax_warnings_for_ignored_variable_filter_prefix
+    ignored_chars = ",wat? lax!"
+    template = Liquid::Template.parse("{{ test#{ignored_chars} | noop }}", error_mode: :lax_warn)
+    assert_equal "works", template.render!('test' => 'works')
+    assert_equal [Liquid::SyntaxError.new("variable filter separator prefixed with ignored characters: #{ignored_chars.inspect}")], template.warnings
+  end
+
+  def test_lax_warnings_for_weird_filter_chars
+    template = Liquid::Template.parse("{{ test | upcase \" prepend: 'it ' || append: ' surprisingly' }}", error_mode: :lax_warn)
+    assert_equal "it WORKS surprisingly", template.render!('test' => 'works')
+    expected_warnings = ['"', '||'].map{ |sep| Liquid::SyntaxError.new("unterminated quote or multiple pipe characters used as a filter separator: #{sep.inspect}") }
+    assert_equal expected_warnings, template.warnings
+  end
+
+  def test_lax_warnings_for_ignored_filter_name_prefix
+    ignored_chars = "'': '"
+    template = Liquid::Template.parse("{{ test | '': 'prepend ' }}", error_mode: :lax_warn)
+    assert_equal "prepend wat", template.render!('test' => 'wat')
+    assert_equal [Liquid::SyntaxError.new("ignored characters before filter name: #{ignored_chars.inspect}")], template.warnings
+  end
 end

test/unit/lexer_unit_test.rb

@@ -36,9 +36,6 @@ class LexerUnitTest < Minitest::Test
 
     tokens = Lexer.new('2foo').tokenize
     assert_equal [[:number, '2'], [:id, 'foo'], [:end_of_string]], tokens
-
-    tokens = Lexer.new('foo-bar--baz-').tokenize
-    assert_equal [[:id, 'foo-bar--baz'], [:dash, "-"], [:end_of_string]], tokens
   end
 
   def test_whitespace