From e1d40c7d89accd91f0d3f9bba0637ee33f708646 Mon Sep 17 00:00:00 2001 From: Michael Angell Date: Tue, 28 Jun 2016 09:15:45 +1000 Subject: [PATCH] Add whitespace control character and associated tests --- lib/liquid.rb | 3 +- lib/liquid/block_body.rb | 21 +- lib/liquid/parse_context.rb | 2 +- test/integration/error_handling_test.rb | 16 + test/integration/trim_mode_test.rb | 525 ++++++++++++++++++++++++ 5 files changed, 563 insertions(+), 4 deletions(-) create mode 100644 test/integration/trim_mode_test.rb diff --git a/lib/liquid.rb b/lib/liquid.rb index 0872b30..7d9da26 100644 --- a/lib/liquid.rb +++ b/lib/liquid.rb @@ -24,6 +24,7 @@ module Liquid ArgumentSeparator = ','.freeze FilterArgumentSeparator = ':'.freeze VariableAttributeSeparator = '.'.freeze + WhitespaceControl = '-'.freeze TagStart = /\{\%/ TagEnd = /\%\}/ VariableSignature = /\(?[\w\-\.\[\]]\)?/ @@ -34,7 +35,7 @@ module Liquid QuotedString = /"[^"]*"|'[^']*'/ QuotedFragment = /#{QuotedString}|(?:[^\s,\|'"]|#{QuotedString})+/o TagAttributes = /(\w+)\s*\:\s*(#{QuotedFragment})/o - AnyStartingTag = /\{\{|\{\%/ + AnyStartingTag = /#{TagStart}|#{VariableStart}/o PartialTemplateParser = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om TemplateParser = /(#{PartialTemplateParser}|#{AnyStartingTag})/om VariableParser = /\[[^\]]+\]|#{VariableSegment}+\??/o diff --git a/lib/liquid/block_body.rb b/lib/liquid/block_body.rb index c375347..89cb6d8 100644 --- a/lib/liquid/block_body.rb +++ b/lib/liquid/block_body.rb @@ -1,7 +1,7 @@ module Liquid class BlockBody - FullToken = /\A#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om - ContentOfVariable = /\A#{VariableStart}(.*)#{VariableEnd}\z/om + FullToken = /\A#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*?)#{WhitespaceControl}?#{TagEnd}\z/om + ContentOfVariable = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om TAGSTART = "{%".freeze VARSTART = "{{".freeze @@ -18,6 +18,7 @@ module Liquid unless token.empty? case when token.start_with?(TAGSTART) + whitespace_handler(token, parse_context) if token =~ FullToken tag_name = $1 markup = $2 @@ -35,9 +36,14 @@ module Liquid raise_missing_tag_terminator(token, parse_context) end when token.start_with?(VARSTART) + whitespace_handler(token, parse_context) @nodelist << create_variable(token, parse_context) @blank = false else + if parse_context.trim_whitespace + token.lstrip! + end + parse_context.trim_whitespace = false @nodelist << token @blank &&= !!(token =~ /\A\s*\z/) end @@ -48,6 +54,17 @@ module Liquid yield nil, nil end + def whitespace_handler(token, parse_context) + if token[2] == WhitespaceControl + previous_token = @nodelist.pop + if previous_token.is_a? String + previous_token.rstrip! + @nodelist << previous_token + end + end + parse_context.trim_whitespace = (token[-3] == WhitespaceControl) + end + def blank? @blank end diff --git a/lib/liquid/parse_context.rb b/lib/liquid/parse_context.rb index fbd7cb6..34986dd 100644 --- a/lib/liquid/parse_context.rb +++ b/lib/liquid/parse_context.rb @@ -1,6 +1,6 @@ module Liquid class ParseContext - attr_accessor :locale, :line_number + attr_accessor :locale, :line_number, :trim_whitespace attr_reader :partial, :warnings, :error_mode def initialize(options = {}) diff --git a/test/integration/error_handling_test.rb b/test/integration/error_handling_test.rb index be232ea..95d65db 100644 --- a/test/integration/error_handling_test.rb +++ b/test/integration/error_handling_test.rb @@ -97,6 +97,22 @@ class ErrorHandlingTest < Minitest::Test assert_match(/Liquid syntax error \(line 4\)/, err.message) end + def test_with_line_numbers_adds_numbers_to_parser_errors_with_whitespace_trim + err = assert_raises(SyntaxError) do + Liquid::Template.parse(%q( + foobar + + {%- "cat" | foobar -%} + + bla + ), + line_numbers: true + ) + end + + assert_match(/Liquid syntax error \(line 4\)/, err.message) + end + def test_parsing_warn_with_line_numbers_adds_numbers_to_lexer_errors template = Liquid::Template.parse(' foobar diff --git a/test/integration/trim_mode_test.rb b/test/integration/trim_mode_test.rb new file mode 100644 index 0000000..ed24fe3 --- /dev/null +++ b/test/integration/trim_mode_test.rb @@ -0,0 +1,525 @@ +require 'test_helper' + +class TrimModeTest < Minitest::Test + include Liquid + + # Make sure the trim isn't applied to standard output + def test_standard_output + text = <<-END_TEMPLATE +
+

+ {{ 'John' }} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+ John +

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_variable_output_with_multiple_blank_lines + text = <<-END_TEMPLATE +
+

+ + + {{- 'John' -}} + + +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

John

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_tag_output_with_multiple_blank_lines + text = <<-END_TEMPLATE +
+

+ + + {%- if true -%} + yes + {%- endif -%} + + +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

yes

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + # Make sure the trim isn't applied to standard tags + def test_standard_tags + whitespace = ' ' + text = <<-END_TEMPLATE +
+

+ {% if true %} + yes + {% endif %} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+#{whitespace} + yes +#{whitespace} +

+
+ END_EXPECTED + assert_template_result(expected, text) + + text = <<-END_TEMPLATE +
+

+ {% if false %} + no + {% endif %} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+#{whitespace} +

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + # Make sure the trim isn't too agressive + def test_no_trim_output + text = '

{{- \'John\' -}}

' + expected = '

John

' + assert_template_result(expected, text) + end + + # Make sure the trim isn't too agressive + def test_no_trim_tags + text = '

{%- if true -%}yes{%- endif -%}

' + expected = '

yes

' + assert_template_result(expected, text) + + text = '

{%- if false -%}no{%- endif -%}

' + expected = '

' + assert_template_result(expected, text) + end + + def test_single_line_outer_tag + text = '

{%- if true %} yes {% endif -%}

' + expected = '

yes

' + assert_template_result(expected, text) + + text = '

{%- if false %} no {% endif -%}

' + expected = '

' + assert_template_result(expected, text) + end + + def test_single_line_inner_tag + text = '

{% if true -%} yes {%- endif %}

' + expected = '

yes

' + assert_template_result(expected, text) + + text = '

{% if false -%} no {%- endif %}

' + expected = '

' + assert_template_result(expected, text) + end + + def test_single_line_post_tag + text = '

{% if true -%} yes {% endif -%}

' + expected = '

yes

' + assert_template_result(expected, text) + + text = '

{% if false -%} no {% endif -%}

' + expected = '

' + assert_template_result(expected, text) + end + + def test_single_line_pre_tag + text = '

{%- if true %} yes {%- endif %}

' + expected = '

yes

' + assert_template_result(expected, text) + + text = '

{%- if false %} no {%- endif %}

' + expected = '

' + assert_template_result(expected, text) + end + + def test_pre_trim_output + text = <<-END_TEMPLATE +
+

+ {{- 'John' }} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

John +

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_pre_trim_tags + text = <<-END_TEMPLATE +
+

+ {%- if true %} + yes + {%- endif %} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+ yes +

+
+ END_EXPECTED + assert_template_result(expected, text) + + text = <<-END_TEMPLATE +
+

+ {%- if false %} + no + {%- endif %} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_post_trim_output + text = <<-END_TEMPLATE +
+

+ {{ 'John' -}} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+ John

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_post_trim_tags + text = <<-END_TEMPLATE +
+

+ {% if true -%} + yes + {% endif -%} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+ yes +

+
+ END_EXPECTED + assert_template_result(expected, text) + + text = <<-END_TEMPLATE +
+

+ {% if false -%} + no + {% endif -%} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_pre_and_post_trim_tags + text = <<-END_TEMPLATE +
+

+ {%- if true %} + yes + {% endif -%} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+ yes +

+
+ END_EXPECTED + assert_template_result(expected, text) + + text = <<-END_TEMPLATE +
+

+ {%- if false %} + no + {% endif -%} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_post_and_pre_trim_tags + text = <<-END_TEMPLATE +
+

+ {% if true -%} + yes + {%- endif %} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+ yes +

+
+ END_EXPECTED + assert_template_result(expected, text) + + whitespace = ' ' + text = <<-END_TEMPLATE +
+

+ {% if false -%} + no + {%- endif %} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+#{whitespace} +

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_trim_output + text = <<-END_TEMPLATE +
+

+ {{- 'John' -}} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

John

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_trim_tags + text = <<-END_TEMPLATE +
+

+ {%- if true -%} + yes + {%- endif -%} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

yes

+
+ END_EXPECTED + assert_template_result(expected, text) + + text = <<-END_TEMPLATE +
+

+ {%- if false -%} + no + {%- endif -%} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_whitespace_trim_output + text = <<-END_TEMPLATE +
+

+ {{- 'John' -}}, + {{- '30' -}} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

John,30

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_whitespace_trim_tags + text = <<-END_TEMPLATE +
+

+ {%- if true -%} + yes + {%- endif -%} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

yes

+
+ END_EXPECTED + assert_template_result(expected, text) + + text = <<-END_TEMPLATE +
+

+ {%- if false -%} + no + {%- endif -%} +

+
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

+
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_complex_trim_output + text = <<-END_TEMPLATE +
+

+ {{- 'John' -}} + {{- '30' -}} +

+ + {{ 'John' -}} + {{- '30' }} + + + {{- 'John' }} + {{ '30' -}} + +
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+

John30

+ + John30 + + John + 30 +
+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_complex_trim + text = <<-END_TEMPLATE +
+ {%- if true -%} + {%- if true -%} +

+ {{- 'John' -}} +

+ {%- endif -%} + {%- endif -%} +
+ END_TEMPLATE + expected = <<-END_EXPECTED +

John

+ END_EXPECTED + assert_template_result(expected, text) + end + + def test_raw_output + whitespace = ' ' + text = <<-END_TEMPLATE +
+ {% raw %} + {%- if true -%} +

+ {{- 'John' -}} +

+ {%- endif -%} + {% endraw %} +
+ END_TEMPLATE + expected = <<-END_EXPECTED +
+#{whitespace} + {%- if true -%} +

+ {{- 'John' -}} +

+ {%- endif -%} +#{whitespace} +
+ END_EXPECTED + assert_template_result(expected, text) + end +end # TrimModeTest