Compare commits

..

1 Commits

Author SHA1 Message Date
Peter Zhu
35570d4ee0 Fix regex for matching endraw tags 2020-11-06 14:54:58 -05:00
21 changed files with 141 additions and 336 deletions

View File

@@ -77,9 +77,6 @@ module Liquid
body.parse(tokens, parse_context) do |end_tag_name, end_tag_params|
@blank &&= body.blank?
# Instrument for bug 1346
Usage.increment("end_tag_params") if end_tag_params && !end_tag_params.empty?
return false if end_tag_name == block_delimiter
raise_tag_never_closed(block_name) unless end_tag_name

View File

@@ -49,14 +49,6 @@ module Liquid
@@method_literals[markup] || parse_context.parse_expression(markup)
end
def self.strict_parse_expression(parse_context, p)
if p.look(:id) && !p.look(:dot, 1) && !p.look(:open_square, 1)
parse_expression(parse_context, p.consume)
else
p.expression
end
end
attr_reader :attachment, :child_condition
attr_accessor :left, :operator, :right

View File

@@ -33,7 +33,7 @@ module Liquid
if LITERALS.key?(markup)
LITERALS[markup]
else
VariableLookup.lax_parse(markup)
VariableLookup.parse(markup)
end
end
end

View File

@@ -9,12 +9,7 @@ module Liquid
@index = 0
end
attr_reader :length, :parentloop
def name
Usage.increment('forloop_drop_name')
@name
end
attr_reader :name, :length, :parentloop
def index
@index + 1

View File

@@ -20,6 +20,7 @@
tag_termination: "Tag '%{token}' was not properly terminated with regexp: %{tag_end}"
variable_termination: "Variable '%{token}' was not properly terminated with regexp: %{tag_end}"
tag_never_closed: "'%{block_name}' tag was never closed"
meta_syntax_error: "Liquid syntax error: #{e.message}"
table_row: "Syntax Error in 'table_row loop' - Valid syntax: table_row [item] in [collection] cols=3"
render: "Syntax error in tag 'render' - Template name must be a quoted string"
argument:

View File

@@ -50,52 +50,53 @@ module Liquid
token = @tokens[@p]
case token[0]
when :id
if Expression::LITERALS.key?(token[1]) && !look(:dot, 1) && !look(:open_square, 1)
Expression::LITERALS[consume]
else
VariableLookup.strict_parse(self)
end
str = consume
str << variable_lookups
when :open_square
VariableLookup.strict_parse(self)
when :string
consume[1..-2]
when :number
num_str = consume
num_str.include?('.') ? num_str.to_f : num_str.to_i
str = consume
str << expression
str << consume(:close_square)
str << variable_lookups
when :string, :number
consume
when :open_round
consume
first = expression
consume(:dotdot)
last = expression
consume(:close_round)
RangeLookup.build(first, last)
"(#{first}..#{last})"
else
raise SyntaxError, "#{token} is not a valid expression"
end
end
def arguments
filter_args = []
keyword_args = nil
loop do
# keyword argument (identifier: expression)
if look(:colon, 1)
keyword_args ||= {}
k = consume(:id)
consume
v = expression
keyword_args[k] = v
else
filter_args << expression
end
break unless consume?(:comma)
def argument
str = +""
# might be a keyword argument (identifier: expression)
if look(:id) && look(:colon, 1)
str << consume << consume << ' '
end
result = [filter_args]
result << keyword_args if keyword_args
result
str << expression
str
end
def variable_lookups
str = +""
loop do
if look(:open_square)
str << consume
str << expression
str << consume(:close_square)
elsif look(:dot)
str << consume
str << consume(:id)
else
break
end
end
str
end
end
end

View File

@@ -5,43 +5,35 @@ module Liquid
def self.parse(start_markup, end_markup)
start_obj = Expression.parse(start_markup)
end_obj = Expression.parse(end_markup)
build(start_obj, end_obj)
end
def self.build(start_obj, end_obj)
if start_obj.respond_to?(:evaluate) || end_obj.respond_to?(:evaluate)
new(start_obj, end_obj)
else
to_integer(start_obj)..to_integer(end_obj)
start_obj.to_i..end_obj.to_i
end
end
def self.to_integer(input)
def initialize(start_obj, end_obj)
@start_obj = start_obj
@end_obj = end_obj
end
def evaluate(context)
start_int = to_integer(context.evaluate(@start_obj))
end_int = to_integer(context.evaluate(@end_obj))
start_int..end_int
end
private
def to_integer(input)
case input
when Integer
input
when NilClass, String, Float
when NilClass, String
input.to_i
else
Utils.to_integer(input)
end
end
attr_reader :start_expr, :end_expr
def initialize(start_expr, end_expr)
@start_expr = start_expr
@end_expr = end_expr
end
def evaluate(context)
start_int = self.class.to_integer(context.evaluate(@start_expr))
end_int = self.class.to_integer(context.evaluate(@end_expr))
start_int..end_int
end
def ==(other)
self.class == other.class && start_expr == other.start_expr && end_expr == other.end_expr
end
end
end

View File

@@ -113,9 +113,10 @@ module Liquid
@variable_name = p.consume(:id)
raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_in") unless p.id?('in')
@collection_name = p.expression
collection_name = p.expression
@collection_name = parse_expression(collection_name)
@name = "#{@variable_name}-#{@collection_name}"
@name = "#{@variable_name}-#{collection_name}"
@reversed = p.id?('reversed')
while p.look(:id) && p.look(:colon, 1)
@@ -123,18 +124,7 @@ module Liquid
raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_attribute")
end
p.consume
case attribute
when 'offset'
@from =
if p.id?('continue')
Usage.increment('for_offset_continue')
:continue
else
p.expression
end
when 'limit'
@limit = p.expression
end
set_attribute(attribute, p.expression)
end
p.consume(:end_of_string)
end
@@ -208,7 +198,6 @@ module Liquid
case key
when 'offset'
@from = if expr == 'continue'
Usage.increment('for_offset_continue')
:continue
else
parse_expression(expr)

View File

@@ -75,10 +75,6 @@ module Liquid
Condition.parse_expression(parse_context, markup)
end
def strict_parse_expression(p)
Condition.strict_parse_expression(parse_context, p)
end
def lax_parse(markup)
expressions = markup.scan(ExpressionsAndOperators)
raise SyntaxError, options[:locale].t("errors.syntax.if") unless expressions.pop =~ Syntax
@@ -118,9 +114,9 @@ module Liquid
end
def parse_comparison(p)
a = strict_parse_expression(p)
a = parse_expression(p.expression)
if (op = p.consume?(:comparison))
b = strict_parse_expression(p)
b = parse_expression(p.expression)
Condition.new(a, op, b)
else
Condition.new(a)

View File

@@ -3,7 +3,7 @@
module Liquid
class Raw < Block
Syntax = /\A\s*\z/
FullTokenPossiblyInvalid = /\A(.*)#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om
EndRawTag = /\A(.*)#{TagStart}\s*endraw.*?#{TagEnd}\z/om
def initialize(tag_name, markup, parse_context)
super
@@ -14,7 +14,7 @@ module Liquid
def parse(tokens)
@body = +''
while (token = tokens.shift)
if token =~ FullTokenPossiblyInvalid && block_delimiter == Regexp.last_match(2)
if token =~ EndRawTag
@body << Regexp.last_match(1) if Regexp.last_match(1) != ""
return
end

View File

@@ -65,15 +65,23 @@ module Liquid
return if p.look(:end_of_string)
@name = p.expression
@name = Expression.parse(p.expression)
while p.consume?(:pipe)
filtername = p.consume(:id)
filterargs = p.consume?(:colon) ? p.arguments : [[]]
@filters << [filtername, *filterargs]
filterargs = p.consume?(:colon) ? parse_filterargs(p) : []
@filters << parse_filter_expressions(filtername, filterargs)
end
p.consume(:end_of_string)
end
def parse_filterargs(p)
# first argument
filterargs = [p.argument]
# followed by comma separated others
filterargs << p.argument while p.consume?(:comma)
filterargs
end
def render(context)
obj = context.evaluate(@name)

View File

@@ -7,66 +7,30 @@ module Liquid
attr_reader :name, :lookups
class << self
def lax_parse(markup)
lookups = markup.scan(VariableParser)
name = lookups.shift
if name =~ SQUARE_BRACKETED
name = Expression.parse(Regexp.last_match(1))
end
command_flags = 0
lookups.each_index do |i|
lookup = lookups[i]
if lookup =~ SQUARE_BRACKETED
lookups[i] = Expression.parse(Regexp.last_match(1))
elsif COMMAND_METHODS.include?(lookup)
command_flags |= 1 << i
end
end
new(name, lookups, command_flags)
end
def strict_parse(p)
if p.look(:id)
name = p.consume
else
p.consume(:open_square)
name = p.expression
p.consume(:close_square)
end
lookups = []
command_flags = 0
loop do
if p.consume?(:open_square)
lookups << p.expression
p.consume(:close_square)
elsif p.consume?(:dot)
lookup = p.consume(:id)
lookups << lookup
if COMMAND_METHODS.include?(lookup)
command_flags |= 1 << (lookups.length - 1)
end
else
break
end
end
new(name, lookups, command_flags)
end
private :new
def self.parse(markup)
new(markup)
end
def initialize(name, lookups, command_flags)
def initialize(markup)
lookups = markup.scan(VariableParser)
name = lookups.shift
if name =~ SQUARE_BRACKETED
name = Expression.parse(Regexp.last_match(1))
end
@name = name
@lookups = lookups
@command_flags = command_flags
@lookups = lookups
@command_flags = 0
@lookups.each_index do |i|
lookup = lookups[i]
if lookup =~ SQUARE_BRACKETED
lookups[i] = Expression.parse(Regexp.last_match(1))
elsif COMMAND_METHODS.include?(lookup)
@command_flags |= 1 << i
end
end
end
def evaluate(context)
@@ -111,19 +75,6 @@ module Liquid
self.class == other.class && state == other.state
end
def to_s
str = name.dup
lookups.each do |lookup|
str +=
if lookup.instance_of?(String)
"['#{lookup}']"
else
"[#{lookup}]"
end
end
str
end
protected
def state

View File

@@ -55,24 +55,4 @@ class BlockTest < Minitest::Test
assert_equal buf.object_id, output.object_id
end
end
def test_instrument_for_bug_1346
calls = []
Liquid::Usage.stub(:increment, ->(name) { calls << name }) do
Liquid::Template.parse("{% for i in (1..2) %}{{ i }}{% endfor {% foo %}")
end
assert_equal(["end_tag_params"], calls)
calls = []
Liquid::Usage.stub(:increment, ->(name) { calls << name }) do
Liquid::Template.parse("{% for i in (1..2) %}{{ i }}{% endfor test %}")
end
assert_equal(["end_tag_params"], calls)
calls = []
Liquid::Usage.stub(:increment, ->(name) { calls << name }) do
Liquid::Template.parse("{% for i in (1..2) %}{{ i }}{% endfor %}")
end
assert_equal([], calls)
end
end

View File

@@ -29,24 +29,17 @@ class ExpressionTest < Minitest::Test
def test_range
assert_equal(1..2, parse_and_eval("(1..2)"))
assert_equal(3..4, parse_and_eval(" ( 3 .. 4 ) "))
assert_equal(0..0, parse_and_eval("('a'..'b')"))
with_error_mode(:strict) do
assert_raises(Liquid::ArgumentError) { parse_and_eval("(1..(1..5))") }
end
end
private
def parse_and_eval(markup, **assigns)
expression =
if Liquid::Template.error_mode == :strict
p = Liquid::Parser.new(markup)
p.expression
else
Liquid::Expression.parse(markup)
end
if Liquid::Template.error_mode == :strict
p = Liquid::Parser.new(markup)
markup = p.expression
p.consume(:end_of_string)
end
expression = Liquid::Expression.parse(markup)
context = Liquid::Context.new(assigns)
context.evaluate(expression)
end

View File

@@ -437,30 +437,4 @@ HERE
assert(context.registers[:for_stack].empty?)
end
def test_instrument_for_offset_continue
assert_usage_increment('for_offset_continue') do
Template.parse('{% for item in items offset:continue %}{{item}}{% endfor %}')
end
assert_usage_increment('for_offset_continue', times: 0) do
Template.parse('{% for item in items offset:2 %}{{item}}{% endfor %}')
end
end
def test_instrument_forloop_drop_name
assigns = { 'items' => [1, 2, 3, 4, 5] }
assert_usage_increment('forloop_drop_name', times: 5) do
Template.parse('{% for item in items %}{{forloop.name}}{% endfor %}').render!(assigns)
end
assert_usage_increment('forloop_drop_name', times: 0) do
Template.parse('{% for item in items %}{{forloop.index}}{% endfor %}').render!(assigns)
end
assert_usage_increment('forloop_drop_name', times: 0) do
Template.parse('{% for item in items %}{{item}}{% endfor %}').render!(assigns)
end
end
end

View File

@@ -12,6 +12,8 @@ class RawTagTest < Minitest::Test
def test_output_in_raw
assert_template_result('{{ test }}', '{% raw %}{{ test }}{% endraw %}')
assert_template_result('test', '{% raw %}test{% endraw{% f %}')
assert_template_result('test', '{% raw %}test{% endraw{% |f %}')
end
def test_open_tag_in_raw

View File

@@ -556,8 +556,4 @@ class TrimModeTest < Minitest::Test
template = Liquid::Template.parse("B\n {%- if true %}{% endif %}", bug_compatible_whitespace_trimming: true)
assert_equal("B", template.render)
end
def test_trim_blank
assert_template_result('foobar', 'foo {{-}} bar')
end
end # TrimModeTest

View File

@@ -77,7 +77,7 @@ class ConditionUnitTest < Minitest::Test
def test_contains_works_on_arrays
@context = Liquid::Context.new
@context['array'] = [1, 2, 3, 4, 5]
array_expr = parse_variable_lookup("array")
array_expr = VariableLookup.new("array")
assert_evaluates_false(array_expr, 'contains', 0)
assert_evaluates_true(array_expr, 'contains', 1)
@@ -91,8 +91,8 @@ class ConditionUnitTest < Minitest::Test
def test_contains_returns_false_for_nil_operands
@context = Liquid::Context.new
assert_evaluates_false(parse_variable_lookup('not_assigned'), 'contains', '0')
assert_evaluates_false(0, 'contains', parse_variable_lookup('not_assigned'))
assert_evaluates_false(VariableLookup.new('not_assigned'), 'contains', '0')
assert_evaluates_false(0, 'contains', VariableLookup.new('not_assigned'))
end
def test_contains_return_false_on_wrong_data_type
@@ -145,20 +145,11 @@ class ConditionUnitTest < Minitest::Test
@context = Liquid::Context.new
@context['one'] = @context['another'] = "gnomeslab-and-or-liquid"
assert_evaluates_true(parse_variable_lookup("one"), '==', parse_variable_lookup("another"))
assert_evaluates_true(VariableLookup.new("one"), '==', VariableLookup.new("another"))
end
private
def parse_variable_lookup(markup)
if Liquid::Template.error_mode == :strict
p = Liquid::Parser.new(markup)
VariableLookup.strict_parse(p)
else
VariableLookup.lax_parse(markup)
end
end
def assert_evaluates_true(left, op, right)
assert(Condition.new(left, op, right).evaluate(@context),
"Evaluated false: #{left} #{op} #{right}")

View File

@@ -47,41 +47,32 @@ class ParserUnitTest < Minitest::Test
def test_expressions
p = Parser.new("hi.there hi?[5].there? hi.there.bob")
assert_equal(VariableLookup.send(:new, 'hi', ['there'], 0), p.expression)
assert_equal(VariableLookup.send(:new, 'hi?', [5, 'there?'], 0), p.expression)
assert_equal(VariableLookup.send(:new, 'hi', ['there', 'bob'], 0), p.expression)
p = Parser.new("nil true false")
assert_nil(p.expression)
assert_equal(true, p.expression)
assert_equal(false, p.expression)
assert_equal('hi.there', p.expression)
assert_equal('hi?[5].there?', p.expression)
assert_equal('hi.there.bob', p.expression)
p = Parser.new("567 6.0 'lol' \"wut\"")
assert_equal(567, p.expression)
assert_equal(6.0, p.expression)
assert_equal('lol', p.expression)
assert_equal('wut', p.expression)
assert_equal('567', p.expression)
assert_equal('6.0', p.expression)
assert_equal("'lol'", p.expression)
assert_equal('"wut"', p.expression)
end
def test_ranges
p = Parser.new("(5..7) (1.5..9.6) (young..old) (hi[5].wat..old)")
assert_equal(5..7, p.expression)
assert_equal(1..9, p.expression)
assert_equal(
RangeLookup.new(VariableLookup.send(:new, 'young', [], 0), VariableLookup.send(:new, 'old', [], 0)),
p.expression
)
assert_equal(
RangeLookup.new(VariableLookup.send(:new, 'hi', [5, "wat"], 0), VariableLookup.send(:new, 'old', [], 0)),
p.expression
)
assert_equal('(5..7)', p.expression)
assert_equal('(1.5..9.6)', p.expression)
assert_equal('(young..old)', p.expression)
assert_equal('(hi[5].wat..old)', p.expression)
end
def test_arguments
p = Parser.new("filter: hi.there[5], keyarg: 7")
assert_equal('filter', p.consume(:id))
assert_equal(':', p.consume(:colon))
assert_equal([[VariableLookup.send(:new, "hi", ["there", 5], 0)], { "keyarg" => 7 }], p.arguments)
assert_equal('hi.there[5]', p.argument)
assert_equal(',', p.consume(:comma))
assert_equal('keyarg: 7', p.argument)
end
def test_invalid_expression

View File

@@ -1,39 +0,0 @@
# frozen_string_literal: true
require 'test_helper'
class VariableLookupUnitTest < Minitest::Test
include Liquid
def test_variable_lookup_parsing
lookup = parse_variable_lookup('a.b.c')
assert_equal('a', lookup.name)
assert_equal(['b', 'c'], lookup.lookups)
lookup = parse_variable_lookup('a[b]')
assert_equal('a', lookup.name)
assert_equal([parse_variable_lookup('b')], lookup.lookups)
end
def test_to_s
lookup = parse_variable_lookup('a.b.c')
assert_equal("a['b']['c']", lookup.to_s)
lookup = parse_variable_lookup('a[b.c].d')
assert_equal("a[b['c']]['d']", lookup.to_s)
lookup = parse_variable_lookup('a["foo.bar"].d')
assert_equal("a['foo.bar']['d']", lookup.to_s)
end
private
def parse_variable_lookup(markup)
if Liquid::Template.error_mode == :strict
p = Liquid::Parser.new(markup)
VariableLookup.strict_parse(p)
else
VariableLookup.lax_parse(markup)
end
end
end

View File

@@ -7,20 +7,20 @@ class VariableUnitTest < Minitest::Test
def test_variable
var = create_variable('hello')
assert_equal(parse_variable_lookup('hello'), var.name)
assert_equal(VariableLookup.new('hello'), var.name)
end
def test_filters
var = create_variable('hello | textileze')
assert_equal(parse_variable_lookup('hello'), var.name)
assert_equal(VariableLookup.new('hello'), var.name)
assert_equal([['textileze', []]], var.filters)
var = create_variable('hello | textileze | paragraph')
assert_equal(parse_variable_lookup('hello'), var.name)
assert_equal(VariableLookup.new('hello'), var.name)
assert_equal([['textileze', []], ['paragraph', []]], var.filters)
var = create_variable(%( hello | strftime: '%Y'))
assert_equal(parse_variable_lookup('hello'), var.name)
assert_equal(VariableLookup.new('hello'), var.name)
assert_equal([['strftime', ['%Y']]], var.filters)
var = create_variable(%( 'typo' | link_to: 'Typo', true ))
@@ -44,11 +44,11 @@ class VariableUnitTest < Minitest::Test
assert_equal([['repeat', [3, 3, 3]]], var.filters)
var = create_variable(%( hello | strftime: '%Y, okay?'))
assert_equal(parse_variable_lookup('hello'), var.name)
assert_equal(VariableLookup.new('hello'), var.name)
assert_equal([['strftime', ['%Y, okay?']]], var.filters)
var = create_variable(%( hello | things: "%Y, okay?", 'the other one'))
assert_equal(parse_variable_lookup('hello'), var.name)
assert_equal(VariableLookup.new('hello'), var.name)
assert_equal([['things', ['%Y, okay?', 'the other one']]], var.filters)
end
@@ -60,24 +60,22 @@ class VariableUnitTest < Minitest::Test
def test_filters_without_whitespace
var = create_variable('hello | textileze | paragraph')
assert_equal(parse_variable_lookup('hello'), var.name)
assert_equal(VariableLookup.new('hello'), var.name)
assert_equal([['textileze', []], ['paragraph', []]], var.filters)
var = create_variable('hello|textileze|paragraph')
assert_equal(parse_variable_lookup('hello'), var.name)
assert_equal(VariableLookup.new('hello'), var.name)
assert_equal([['textileze', []], ['paragraph', []]], var.filters)
var = create_variable("hello|replace:'foo','bar'|textileze")
assert_equal(parse_variable_lookup('hello'), var.name)
assert_equal(VariableLookup.new('hello'), var.name)
assert_equal([['replace', ['foo', 'bar']], ['textileze', []]], var.filters)
end
def test_symbol
with_error_mode(:lax) do
var = create_variable("http://disney.com/logo.gif | image: 'med' ", error_mode: :lax)
assert_equal(parse_variable_lookup('http://disney.com/logo.gif'), var.name)
assert_equal([['image', ['med']]], var.filters)
end
var = create_variable("http://disney.com/logo.gif | image: 'med' ", error_mode: :lax)
assert_equal(VariableLookup.new('http://disney.com/logo.gif'), var.name)
assert_equal([['image', ['med']]], var.filters)
end
def test_string_to_filter
@@ -107,8 +105,8 @@ class VariableUnitTest < Minitest::Test
end
def test_dashes
assert_equal(parse_variable_lookup('foo-bar'), create_variable('foo-bar').name)
assert_equal(parse_variable_lookup('foo-bar-2'), create_variable('foo-bar-2').name)
assert_equal(VariableLookup.new('foo-bar'), create_variable('foo-bar').name)
assert_equal(VariableLookup.new('foo-bar-2'), create_variable('foo-bar-2').name)
with_error_mode :strict do
assert_raises(Liquid::SyntaxError) { create_variable('foo - bar') }
@@ -124,18 +122,18 @@ class VariableUnitTest < Minitest::Test
def test_string_dot
var = create_variable(%( test.test ))
assert_equal(parse_variable_lookup('test.test'), var.name)
assert_equal(VariableLookup.new('test.test'), var.name)
end
def test_filter_with_keyword_arguments
var = create_variable(%( hello | things: greeting: "world", farewell: 'goodbye'))
assert_equal(parse_variable_lookup('hello'), var.name)
assert_equal(VariableLookup.new('hello'), var.name)
assert_equal([['things', [], { 'greeting' => 'world', 'farewell' => 'goodbye' }]], var.filters)
end
def test_lax_filter_argument_parsing
var = create_variable(%( number_of_comments | pluralize: 'comment': 'comments' ), error_mode: :lax)
assert_equal(parse_variable_lookup('number_of_comments'), var.name)
assert_equal(VariableLookup.new('number_of_comments'), var.name)
assert_equal([['pluralize', ['comment', 'comments']]], var.filters)
end
@@ -152,17 +150,14 @@ class VariableUnitTest < Minitest::Test
assert_equal(" name_of_variable | upcase ", var.raw)
end
private
def parse_variable_lookup(markup)
if Liquid::Template.error_mode == :strict
p = Liquid::Parser.new(markup)
VariableLookup.strict_parse(p)
else
VariableLookup.lax_parse(markup)
end
def test_variable_lookup_interface
lookup = VariableLookup.new('a.b.c')
assert_equal('a', lookup.name)
assert_equal(['b', 'c'], lookup.lookups)
end
private
def create_variable(markup, options = {})
Variable.new(markup, ParseContext.new(options))
end