Compare commits

..

13 Commits

Author SHA1 Message Date
Mike Angell
8f68cffdf1 Resolve failing rubocop issues 2019-08-29 00:45:38 +10:00
Mike Angell
dd27d0fd1d Merge pull request #1133 from Shopify/liquid-tag-fixes
Bugfix for new Liquid tag
2019-08-29 00:36:13 +10:00
Mike Angell
7a26e6b3d8 Merge pull request #1131 from Shopify/bump-ruby-2-4
Rubocop upgrade, Ruby 2.4 minimum and TruffleRuby
2019-08-29 00:33:42 +10:00
Mike Angell
cf4e77ab0c Merge branch 'master' into bump-ruby-2-4 2019-08-29 00:24:45 +10:00
Mike Angell
7bae55dd39 Bugfix for new Liquid tag 2019-08-28 23:39:19 +10:00
Tobias Lütke
0ce8aef229 Merge pull request #1103 from ashmaroli/ci-profile-memory
Add a CI job to profile memory usage of commit
2019-08-27 15:11:55 -04:00
Tobias Lütke
6eab595fae Merge pull request #1086 from Shopify/liquid-tag
Add {% liquid %} and {% echo %} tags
2019-08-27 15:10:20 -04:00
Mike Angell
b16b109a80 Bump Minimum version to 2.4 and bump Rubocop 2019-08-28 00:31:44 +10:00
Ashwin Maroli
ab698191b9 Add a CI job to profile memory usage of commit 2019-05-17 22:47:05 +05:30
Justin Li
7dc488a73b Simplifications from review 2019-04-09 15:19:47 -04:00
Justin Li
e6ed804ca5 Fix line number tracking after a non-empty blank token 2019-04-08 18:43:09 -04:00
Justin Li
951abb67ee Remove {% local %} tag 2019-04-08 18:34:39 -04:00
Justin Li
8d1cd41453 Add {% liquid %}, {% echo %}, and {% local %} tags 2019-04-01 20:08:38 -04:00
28 changed files with 423 additions and 782 deletions

View File

@@ -2,9 +2,15 @@ inherit_from:
- .rubocop_todo.yml
- ./.rubocop_todo.yml
require: rubocop-performance
Performance:
Enabled: true
AllCops:
Exclude:
- 'performance/shopify/*'
- 'vendor/bundle/**/*'
- 'pkg/**'
Metrics/BlockNesting:
@@ -79,9 +85,6 @@ Style/TrailingCommaInArrayLiteral:
Style/TrailingCommaInHashLiteral:
Enabled: false
Layout/IndentHash:
EnforcedStyle: consistent
Style/FormatString:
Enabled: false
@@ -106,9 +109,6 @@ Style/RegexpLiteral:
Style/SymbolLiteral:
Enabled: false
Performance/Count:
Enabled: false
Naming/ConstantName:
Enabled: false

View File

@@ -1,6 +1,6 @@
# This configuration was generated by
# `rubocop --auto-gen-config`
# on 2019-04-22 19:11:24 -0400 using RuboCop version 0.53.0.
# on 2019-08-29 00:43:36 +1000 using RuboCop version 0.74.0.
# The point is for the user to remove these configuration records
# one by one as the offenses are removed from the code base.
# Note that changes in the inspected code, or installation of new
@@ -8,16 +8,67 @@
# Offense count: 1
# Cop supports --auto-correct.
# Configuration parameters: Include, TreatCommentsAsGroupSeparators.
# Configuration parameters: TreatCommentsAsGroupSeparators, Include.
# Include: **/*.gemspec
Gemspec/OrderedDependencies:
Exclude:
- 'liquid.gemspec'
# Offense count: 1
# Configuration parameters: Include.
# Include: **/*.gemspec,
Gemspec/RequiredRubyVersion:
Exclude:
- 'liquid.gemspec'
# Offense count: 124
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle, IndentationWidth.
# SupportedStyles: with_first_argument, with_fixed_indentation
Layout/AlignArguments:
Enabled: false
# Offense count: 7
# Cop supports --auto-correct.
# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
# SupportedHashRocketStyles: key, separator, table
# SupportedColonStyles: key, separator, table
# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit
Layout/AlignHash:
Exclude:
- 'lib/liquid/condition.rb'
- 'lib/liquid/expression.rb'
- 'test/unit/context_unit_test.rb'
# Offense count: 6
# Cop supports --auto-correct.
Layout/ClosingHeredocIndentation:
Exclude:
- 'test/integration/tags/for_tag_test.rb'
# Offense count: 27
# Cop supports --auto-correct.
Layout/EmptyLineAfterGuardClause:
Exclude:
- 'lib/liquid/block.rb'
- 'lib/liquid/block_body.rb'
- 'lib/liquid/context.rb'
- 'lib/liquid/drop.rb'
- 'lib/liquid/lexer.rb'
- 'lib/liquid/parser.rb'
- 'lib/liquid/standardfilters.rb'
- 'lib/liquid/strainer.rb'
- 'lib/liquid/tags/for.rb'
- 'lib/liquid/tags/if.rb'
- 'lib/liquid/tags/include.rb'
- 'lib/liquid/utils.rb'
- 'lib/liquid/variable.rb'
- 'lib/liquid/variable_lookup.rb'
# Offense count: 5
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
# SupportedStyles: auto_detection, squiggly, active_support, powerpack, unindent
# SupportedStyles: squiggly, active_support, powerpack, unindent
Layout/IndentHeredoc:
Exclude:
- 'test/integration/tags/for_tag_test.rb'
@@ -32,6 +83,13 @@ Layout/MultilineMethodCallBraceLayout:
- 'test/integration/error_handling_test.rb'
- 'test/unit/strainer_unit_test.rb'
# Offense count: 1
# Cop supports --auto-correct.
# Configuration parameters: AllowForAlignment.
Layout/SpaceAroundOperators:
Exclude:
- 'lib/liquid/condition.rb'
# Offense count: 2
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
@@ -54,27 +112,25 @@ Metrics/AbcSize:
Metrics/CyclomaticComplexity:
Max: 13
# Offense count: 112
# Configuration parameters: CountComments.
# Offense count: 118
# Configuration parameters: CountComments, ExcludedMethods.
Metrics/MethodLength:
Max: 38
# Offense count: 8
# Offense count: 9
Metrics/PerceivedComplexity:
Max: 11
# Offense count: 52
# Configuration parameters: Blacklist.
# Blacklist: END, (?-mix:EO[A-Z]{1})
Naming/HeredocDelimiterNaming:
# Offense count: 1
# Cop supports --auto-correct.
# Configuration parameters: PreferredName.
Naming/RescuedExceptionsVariableName:
Exclude:
- 'test/integration/assign_test.rb'
- 'test/integration/capture_test.rb'
- 'test/integration/trim_mode_test.rb'
- 'lib/liquid/context.rb'
# Offense count: 23
# Offense count: 20
# Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames.
# AllowedNames: io, id
# AllowedNames: io, id, to, by, on, in, at, ip, db
Naming/UncommunicativeMethodParamName:
Exclude:
- 'example/server/example_servlet.rb'
@@ -82,15 +138,23 @@ Naming/UncommunicativeMethodParamName:
- 'lib/liquid/context.rb'
- 'lib/liquid/standardfilters.rb'
- 'lib/liquid/tags/if.rb'
- 'lib/liquid/utils.rb'
- 'lib/liquid/variable.rb'
- 'test/integration/filter_test.rb'
- 'test/integration/standard_filter_test.rb'
- 'test/integration/tags/for_tag_test.rb'
- 'test/integration/template_test.rb'
- 'test/unit/condition_unit_test.rb'
# Offense count: 12
# Offense count: 5
# Configuration parameters: EnforcedStyle.
# SupportedStyles: inline, group
Style/AccessModifierDeclarations:
Exclude:
- 'lib/liquid/block_body.rb'
- 'lib/liquid/tag.rb'
- 'lib/liquid/tags/include.rb'
- 'test/unit/strainer_unit_test.rb'
# Offense count: 10
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
# SupportedStyles: prefer_alias, prefer_alias_method
@@ -117,15 +181,9 @@ Style/ConditionalAssignment:
- 'lib/liquid/errors.rb'
# Offense count: 1
Style/DateTime:
Exclude:
- 'test/unit/context_unit_test.rb'
# Offense count: 2
# Cop supports --auto-correct.
Style/EmptyCaseCondition:
Exclude:
- 'lib/liquid/block_body.rb'
- 'lib/liquid/lexer.rb'
# Offense count: 5
@@ -163,6 +221,13 @@ Style/FormatStringToken:
- 'test/integration/filter_test.rb'
- 'test/integration/hash_ordering_test.rb'
# Offense count: 106
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
# SupportedStyles: always, never
Style/FrozenStringLiteralComment:
Enabled: false
# Offense count: 14
# Configuration parameters: MinBodyLength.
Style/GuardClause:
@@ -180,6 +245,13 @@ Style/GuardClause:
- 'lib/liquid/variable.rb'
- 'test/unit/tokenizer_unit_test.rb'
# Offense count: 53
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
# SupportedStyles: literals, strict
Style/MutableConstant:
Enabled: false
# Offense count: 1
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle, MinBodyLength.
@@ -188,9 +260,9 @@ Style/Next:
Exclude:
- 'lib/liquid/tags/for.rb'
# Offense count: 4
# Offense count: 13
# Cop supports --auto-correct.
# Configuration parameters: AutoCorrect, EnforcedStyle.
# Configuration parameters: AutoCorrect, EnforcedStyle, IgnoredMethods.
# SupportedStyles: predicate, comparison
Style/NumericPredicate:
Exclude:
@@ -199,6 +271,8 @@ Style/NumericPredicate:
- 'lib/liquid/forloop_drop.rb'
- 'lib/liquid/standardfilters.rb'
- 'lib/liquid/tablerowloop_drop.rb'
- 'test/integration/standard_filter_test.rb'
- 'test/integration/template_test.rb'
# Offense count: 14
# Cop supports --auto-correct.
@@ -216,6 +290,16 @@ Style/RedundantSelf:
Exclude:
- 'lib/liquid/strainer.rb'
# Offense count: 3
# Cop supports --auto-correct.
# Configuration parameters: ConvertCodeThatCanStartToReturnNil, Whitelist.
# Whitelist: present?, blank?, presence, try, try!
Style/SafeNavigation:
Exclude:
- 'lib/liquid/drop.rb'
- 'lib/liquid/strainer.rb'
- 'lib/liquid/tokenizer.rb'
# Offense count: 9
# Cop supports --auto-correct.
# Configuration parameters: AllowAsExpressionSeparator.
@@ -253,8 +337,9 @@ Style/WhileUntilModifier:
Exclude:
- 'lib/liquid/tags/case.rb'
# Offense count: 648
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
# Offense count: 665
# Cop supports --auto-correct.
# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
# URISchemes: http, https
Metrics/LineLength:
Max: 294

View File

@@ -1,29 +1,26 @@
language: ruby
rvm:
- 2.1
- 2.2
- 2.3
- 2.4
- 2.5
- &latest_ruby 2.6
- 2.7
- ruby-head
- jruby-head
# - rbx-2
- truffleruby
sudo: false
addons:
apt:
packages:
- libgmp3-dev
matrix:
include:
- rvm: *latest_ruby
script: bundle exec rake memory_profile:run
name: Profiling Memory Usage
allow_failures:
- rvm: ruby-head
- rvm: jruby-head
- rvm: truffleruby
install:
- bundle install
cache: bundler
script: bundle exec rake

14
Gemfile
View File

@@ -5,23 +5,21 @@ end
gemspec
group :benchmark, :test do
gem 'benchmark-ips'
gem 'memory_profiler'
gem 'terminal-table'
install_if -> { RUBY_PLATFORM !~ /mingw|mswin|java/ } do
install_if -> { RUBY_PLATFORM !~ /mingw|mswin|java/ && RUBY_ENGINE != 'truffleruby' } do
gem 'stackprof'
end
end
group :test do
gem 'rubocop', '~> 0.53.0'
gem 'awesome_print'
gem 'pry'
gem 'byebug'
gem 'rubocop', '~> 0.74.0', require: false
gem 'rubocop-performance', require: false
platform :mri do
gem 'liquid-c', github: 'Shopify/liquid-c', ref: '9168659de45d6d576fce30c735f857e597fa26f6'
platform :mri, :truffleruby do
gem 'liquid-c', github: 'Shopify/liquid-c', ref: 'liquid-tag'
end
end

View File

@@ -19,8 +19,10 @@ task :warn_test do
end
task :rubocop do
require 'rubocop/rake_task'
RuboCop::RakeTask.new
if RUBY_ENGINE == 'ruby'
require 'rubocop/rake_task'
RuboCop::RakeTask.new
end
end
desc 'runs test suite with both strict and lax parsers'
@@ -32,8 +34,8 @@ task :test do
Rake::Task['base_test'].reenable
Rake::Task['base_test'].invoke
if RUBY_ENGINE == 'ruby'
ENV['LIQUID-C'] = '1'
if RUBY_ENGINE == 'ruby' || RUBY_ENGINE == 'truffleruby'
ENV['LIQUID_C'] = '1'
ENV['LIQUID_PARSER_MODE'] = 'lax'
Rake::Task['base_test'].reenable
@@ -45,17 +47,6 @@ task :test do
end
end
desc 'runs the test suite using the superfluid compiler'
Rake::TestTask.new(:test_superfluid) do |t|
t.libs << '.' << 'lib' << 'test'
t.test_files = FileList['test/integration/**/*_test.rb']
t.verbose = false
ENV['LIQUID_PARSER_MODE'] = 'strict'
ENV['LIQUID-C'] = '1'
ENV['SUPERFLUID'] = '1'
end
task gem: :build
task :build do
system "gem build liquid.gemspec"
@@ -82,11 +73,6 @@ namespace :benchmark do
task :strict do
ruby "./performance/benchmark.rb strict"
end
desc "Run the liquid benchmark with strict parsing"
task :superfluid do
ruby "./performance/benchmark.rb superfluid"
end
end
namespace :profile do

View File

@@ -1,6 +1,7 @@
module Liquid
class BlockBody
FullToken = /\A#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
LiquidTagToken = /\A\s*(\w+)\s*(.*?)\z/o
FullToken = /\A#{TagStart}#{WhitespaceControl}?(\s*)(\w+)(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
ContentOfVariable = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
WhitespaceOrNothing = /\A\s*\z/
TAGSTART = "{%".freeze
@@ -13,8 +14,42 @@ module Liquid
@blank = true
end
def parse(tokenizer, parse_context)
def parse(tokenizer, parse_context, &block)
parse_context.line_number = tokenizer.line_number
if tokenizer.for_liquid_tag
parse_for_liquid_tag(tokenizer, parse_context, &block)
else
parse_for_document(tokenizer, parse_context, &block)
end
end
private def parse_for_liquid_tag(tokenizer, parse_context)
while token = tokenizer.shift
unless token.empty? || token =~ WhitespaceOrNothing
unless token =~ LiquidTagToken
# line isn't empty but didn't match tag syntax, yield and let the
# caller raise a syntax error
return yield token, token
end
tag_name = $1
markup = $2
unless tag = registered_tags[tag_name]
# end parsing if we reach an unknown tag and let the caller decide
# determine how to proceed
return yield tag_name, markup
end
new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
@blank &&= new_tag.blank?
@nodelist << new_tag
end
parse_context.line_number = tokenizer.line_number
end
yield nil, nil
end
private def parse_for_document(tokenizer, parse_context, &block)
while token = tokenizer.shift
next if token.empty?
case
@@ -23,9 +58,20 @@ module Liquid
unless token =~ FullToken
raise_missing_tag_terminator(token, parse_context)
end
tag_name = $1
markup = $2
# fetch the tag from registered blocks
tag_name = $2
markup = $4
if parse_context.line_number
# newlines inside the tag should increase the line number,
# particularly important for multiline {% liquid %} tags
parse_context.line_number += $1.count("\n".freeze) + $3.count("\n".freeze)
end
if tag_name == 'liquid'.freeze
liquid_tag_tokenizer = Tokenizer.new(markup, line_number: parse_context.line_number, for_liquid_tag: true)
next parse_for_liquid_tag(liquid_tag_tokenizer, parse_context, &block)
end
unless tag = registered_tags[tag_name]
# end parsing if we reach an unknown tag and let the caller decide
# determine how to proceed
@@ -97,7 +143,7 @@ module Liquid
end
idx += 1
context.raise_if_resource_limits_reached(output.bytesize - previous_output_size)
raise_if_resource_limits_reached(context, output.bytesize - previous_output_size)
end
output
@@ -114,6 +160,12 @@ module Liquid
output << context.handle_error(e, line_number)
end
def raise_if_resource_limits_reached(context, length)
context.resource_limits.render_length += length
return unless context.resource_limits.reached?
raise MemoryError.new("Memory limits exceeded".freeze)
end
def create_variable(token, parse_context)
token.scan(ContentOfVariable) do |content|
markup = content.first

View File

@@ -29,7 +29,7 @@ module Liquid
@@operators
end
attr_reader :attachment, :child_condition, :child_relation
attr_reader :attachment, :child_condition
attr_accessor :left, :operator, :right
def initialize(left = nil, operator = nil, right = nil)
@@ -81,6 +81,10 @@ module Liquid
"#<Condition #{[@left, @operator, @right].compact.join(' '.freeze)}>"
end
protected
attr_reader :child_relation
private
def equal_variables(left, right)

View File

@@ -37,16 +37,6 @@ module Liquid
@global_filter = nil
end
def raise_argument_error(message)
raise Liquid::ArgumentError, message
end
def raise_if_resource_limits_reached(length)
resource_limits.render_length += length
return unless resource_limits.reached?
raise MemoryError.new("Memory limits exceeded".freeze)
end
def warnings
@warnings ||= []
end

View File

@@ -15,8 +15,6 @@ module Liquid
@end_obj = end_obj
end
attr_reader :start_obj, :end_obj
def evaluate(context)
start_int = to_integer(context.evaluate(@start_obj))
end_int = to_integer(context.evaluate(@end_obj))

View File

@@ -1,632 +0,0 @@
require 'ap'
require 'pry'
require 'stackprof'
AwesomePrint.defaults = {
raw: true
}
module Liquid
Liquid::BlockBody.class_eval do
def render_to_output_buffer(context, output)
ruby = Compiler.compile(@nodelist)
if false
puts
puts "--------------------------- GENERATED RUBY -"
line_number = 1
puts(ruby.lines.map do |line|
"#{line_number}\t#{line}".tap { line_number += 1}
end)
puts "--------------------------- /GENERATED RUBY -"
end
instructions = RubyVM::InstructionSequence.compile(ruby)
output_io = StringIO.new
instructions.eval.call(output_io, context, Condition)
output << output_io.string
end
end
class SuperfluidError < Exception
end
class Output
attr_reader :string, :indent_level
def initialize(initial_indent)
@string = ''.dup
@indent_level = initial_indent
@indent_str = " " * initial_indent * 2
end
def line(string)
@string << @indent_str << string << "\n"
end
def echo(string)
output << "liquid_out.write(to_output(#{string}))"
end
def indent(&block)
output.indent(&block)
end
def indent
@indent_level += 1
@indent_str = " " * @indent_level * 2
yield
@indent_level -= 1
@indent_str = " " * @indent_level * 2
end
end
class Compiler
class << self
def compile(template)
compiler = new
compiler.compile(template)
compiler.ruby
end
end
def initialize
@variables = Set.new
@output = Output.new(2)
@blank = false
end
def ruby
[
header,
@output.string,
trailer
].join("\n")
end
def header
<<~RUBY
module Warning
def warn(*)
end
end
class ForloopDrop
def initialize(name, length, parentloop)
@name = name
@length = length
@parentloop = parentloop
@index = 0
end
attr_accessor :parentloop
def [](value)
case value
when "length"
@length
when "name"
@name
when "index"
@index + 1
when "index0"
@index
when "rindex"
@length - @index
when "rindex0"
@length - @index - 1
when "first"
@index == 0
when "last"
@index == @length - 1
when "parentloop"
@parentloop
end
end
def to_liquid
self
end
def key?(*)
true
end
private
def increment!
@index += 1
end
end
def slice_collection(collection, from, limit)
to = if limit.nil?
nil
else
limit + from
end
if (from != 0 || !to.nil?) && collection.respond_to?(:load_slice)
collection.load_slice(from, to)
else
slice_collection_using_each(collection, from, to)
end
end
def slice_collection_using_each(collection, from, to)
segments = []
index = 0
if collection.is_a?(String)
return collection.empty? ? [] : [collection]
end
return [] unless collection.respond_to?(:each)
collection.each do |item|
if to && to <= index
break
end
if from <= index
segments << item
end
index += 1
end
segments
end
def apply_operator(left, operator, right)
if left.respond_to?(operator) && right.respond_to?(operator) && !left.is_a?(Hash) && !right.is_a?(Hash)
begin
left.send(operator, right)
rescue ::ArgumentError => e
raise @context.raise_argument_error(e.message)
end
end
end
def contains?(left, right)
if left && right && left.respond_to?(:include?)
right = right.to_s if left.is_a?(String)
left.include?(right)
else
false
end
end
class GlobalVariableLookup
def method_missing(method_name, *)
nil
end
def to_output(value)
output = if value.is_a?(Array)
value.join
elsif value == nil
else
value.to_s
end
@context.apply_global_filter(output)
end
def run(liquid_out, context, condition)
@condition = condition
@context = context
@for_offsets = {}
@cycle_values = {}
@context.registers[:for_stack] = []
@context.registers[:cycle] ||= {}
@if_changed_last = nil
@prev_output_size = 0
#{hoisted_variables}
RUBY
end
def trailer
<<~RUBY
end
end
GlobalVariableLookup.new.method(:run)
RUBY
end
def hoisted_variables
@variables.map do |variable|
normal_name = unvar(variable)
"#{variable} = @context.find_variable(#{normal_name.inspect}, raise_on_not_found: false)"
end.join("\n")
end
def compile(node)
old_blank = @blank
@blank = if node.respond_to?(:blank?)
node.blank?
elsif !(node.is_a?(String) && node =~ /\A\s*\z/)
false
else
@blank
end
case node
when Liquid::Document, Liquid::BlockBody
node.nodelist.collect(&method(:compile))
when Array
node.collect(&method(:compile))
when Liquid::Variable
compile_variable(node)
when Liquid::For
compile_for(node)
when Liquid::If
compile_if(node)
when Liquid::Ifchanged
compile_if_changed(node)
when Liquid::Template
compile(node.root)
when Liquid::Assign
compile_assign(node)
when Liquid::Case
compile_case(node)
when Liquid::Capture
compile_capture(node)
when String
compile_echo_literal(node)
when Liquid::Break
line "break" if @in_loop
when Liquid::Continue
line "next" if @in_loop
when Liquid::Cycle
compile_cycle(node)
when Liquid::Raw
compile_raw(node)
when Liquid::Increment
compile_increment(node)
when Liquid::Decrement
compile_decrement(node)
when Liquid::Comment
else
raise SuperfluidError, "Unknown node type #{node.inspect}"
end
@blank = old_blank
end
def compile_for(node)
variable_name = node.variable_name
collection_name = node.collection_name
iter_target_expr = case collection_name
when Liquid::VariableLookup
make_variable_lookup_expr(collection_name)
when Range
collection_name
when Liquid::RangeLookup
start_expr = make_variable_expr(collection_name.start_obj)
end_expr = make_variable_expr(collection_name.end_obj)
line "start = #{start_expr}"
line "@context.raise_argument_error('bad value for range') unless start.respond_to?(:to_i)"
line "start = #{start_expr}.to_i"
line "finish = #{end_expr}"
line "@context.raise_argument_error('bad value for range') unless finish.respond_to?(:to_i)"
line "finish = #{end_expr}.to_i"
"start..finish"
when Liquid::Expression::MethodLiteral
'[]'
else
raise SuperfluidError, "Unknown iteration target: #{collection_name.inspect}"
end
from_expr = if node.from == :continue
"@for_offsets['#{node.name}'].to_i"
elsif node.from
make_variable_expr(node.from)
else
'0'
end
limit_expr = node.limit ? make_variable_expr(node.limit) : 'nil'
line "from = #{from_expr}"
line "limit = #{limit_expr}"
line "@context.raise_argument_error('invalid integer') unless from.is_a?(Integer)"
line "@context.raise_argument_error('invalid integer') unless !limit || limit.is_a?(Integer)"
line "segment = slice_collection(#{iter_target_expr}, from, limit)"
line "segment.reverse!" if node.reversed
forloop = var('forloop')
hoist_var('forloop')
line "#{forloop} = ForloopDrop.new('#{node.name}', segment.length, #{forloop})"
line "if segment.any?"
indent do
line "segment.each do |#{var(variable_name)}|"
indent do
line "@context['forloop'] = #{forloop}"
old_in_loop = @in_loop
compile(node.for_block)
@in_loop = old_in_loop
line "#{forloop}.send(:increment!)"
end
line "end"
end
line "else"
indent do
compile(node.else_block) if node.else_block
end
line "end"
line "@for_offsets['#{node.name}'] = from + segment.length"
line "#{forloop} = #{forloop}.parentloop"
end
def compile_if(node)
if_condition = node.blocks.first
line "if #{make_condition_expr(if_condition)}"
indent { if_condition.attachment.nodelist.each(&method(:compile)) }
node.blocks.drop(1).each do |condition|
if condition.left != nil
line "elsif #{make_condition_expr(condition)}"
else
line "else"
end
indent { condition.attachment.nodelist.each(&method(:compile)) }
end
line "end"
end
def compile_if_changed(node)
line "if_changed = lambda do |; liquid_out|"
indent do
line "liquid_out = StringIO.new"
node.nodelist.each(&method(:compile))
line "liquid_out.string"
end
line "end.call"
line "if if_changed != @if_changed_last"
indent { echo "if_changed" }
line "end"
line "@if_changed_last = if_changed"
end
def compile_capture(node)
line "#{var(node.to)} = lambda do |; liquid_out|"
hoist_var(node.to)
indent do
line "liquid_out = StringIO.new"
node.nodelist.each(&method(:compile))
line "liquid_out.string"
end
line "end.call"
end
def make_condition_expr(node)
condition = make_sub_condition_expr(node)
if node.child_condition
"(#{condition} #{node.child_relation} #{make_condition_expr(node.child_condition)})"
else
condition
end
end
def make_sub_condition_expr(node)
return make_variable_expr(node.left) unless node.operator
operator = node.operator
operator = "!=" if operator == "<>"
if operator == "=="
if node.left.is_a?(Liquid::Expression::MethodLiteral) &&
node.right.is_a?(Liquid::Expression::MethodLiteral)
return "false"
elsif node.right.is_a?(Liquid::Expression::MethodLiteral)
target = make_variable_expr(node.left)
message = node.right.method_name.inspect
return "#{target}.respond_to?(#{message}) ? #{target}.send(#{message}) : nil"
elsif node.left.is_a?(Liquid::Expression::MethodLiteral)
target = make_variable_expr(node.right)
message = node.left.method_name.inspect
return "#{target}.respond_to?(#{message}) ? #{target}.send(#{message}) : nil"
end
end
left = make_variable_expr(node.left)
right = make_variable_expr(node.right)
case operator
when "contains"
"contains?(#{left}, #{right})"
else
"apply_operator(#{left}, #{operator.inspect}, #{right})"
end
end
def compile_case(node)
line 'if false' # HACK
else_nodes, if_nodes = node.blocks.partition { |n| n.is_a?(Liquid::ElseCondition) }
raise SuperfluidError, 'Too many else nodes' if else_nodes.count > 1
else_node = else_nodes.first
if_nodes.each do |condition|
left = make_variable_expr(condition.left)
right = make_variable_expr(condition.right)
line "elsif #{left} #{condition.operator} #{right}"
indent { condition.attachment.nodelist.each(&method(:compile)) }
end
if else_node
line "else"
indent { else_node.attachment.nodelist.each(&method(:compile)) }
end
line "end"
end
def compile_cycle(node)
key = node.name
key = key.name if key.is_a?(Liquid::VariableLookup)
line "key = #{key.inspect}"
line "iteration = context.registers[:cycle][key].to_i"
line "@cycle_values[key] ||= #{node.variables}"
line "val = @cycle_values[key][iteration]"
echo 'val'
line "context.registers[:cycle][key] = (iteration + 1) % #{node.variables.size}"
end
def compile_raw(node)
echo "#{node.body.inspect}"
end
def compile_increment(node)
line "value = context.environments.first[#{node.variable.inspect}] ||= 0"
line "@context.environments.first[#{node.variable.inspect}] = value + 1"
echo "value"
end
def compile_decrement(node)
line "value = context.environments.first[#{node.variable.inspect}] ||= 0"
line "value -= 1"
line "@context.environments.first[#{node.variable.inspect}] = value"
echo "value"
end
def compile_echo_literal(node)
echo node.inspect
end
def compile_variable(variable)
echo make_variable_expr(variable)
end
def compile_assign(node)
from_expr = case node.from
when Liquid::Variable
make_variable_expr(node.from)
else
raise SuperfluidError, "Unknown assignment `from`: #{node.from.inspect}"
end
line "#{var(node.to)} = #{from_expr}"
hoist_var(node.to)
end
def make_variable_expr(variable)
case variable
when Liquid::Variable
base_expression = case variable.name
when TrueClass, FalseClass, Numeric, String
variable.name.inspect
when Liquid::VariableLookup
make_variable_lookup_expr(variable.name)
when NilClass, Liquid::Expression::MethodLiteral
'nil'
else
raise SuperfluidError, "Invalid variable name: #{variable.name.inspect}"
derp "Bad var name", variable.name
end
variable.filters.inject(base_expression) do |inner, (filter_name, positional_args, keyword_args)|
filter_args = positional_args.map(&method(:make_variable_expr))
if keyword_args
filter_args << "{ " + keyword_args
.transform_values(&method(:make_variable_expr))
.collect { |(key, value)| "#{key.inspect} => #{value}" }
.join(", ") + " }"
end
"context.strainer.invoke(#{filter_name.inspect}, #{inner}, *[#{filter_args.join(", ")}])"
end
when Liquid::VariableLookup
make_variable_lookup_expr(variable)
when TrueClass, FalseClass, Numeric, String
variable.inspect
when NilClass
'nil'
else
raise SuperfluidError, "Unknown expression type: #{variable.inspect}"
end
end
def make_variable_lookup_expr(variable_lookup)
base_expr = var(variable_lookup.name)
hoist_var(variable_lookup.name)
return base_expr if variable_lookup.lookups.empty?
expr = Output.new(output.indent_level)
expr.line "(begin"
expr.indent do
expr.line "inner = #{base_expr}"
variable_lookup.lookups.each_with_index do |lookup, i|
lookup_expr = lookup.inspect
expr.line "inner = if inner.respond_to?(:[]) && ((inner.respond_to?(:key?) && inner.key?(#{lookup_expr})) || (inner.respond_to?(:fetch) && #{lookup_expr}.is_a?(Integer)))"
expr.indent do
expr.line "inner[#{lookup_expr}].to_liquid"
end
if variable_lookup.command_flags & (1 << i) != 0
expr.line "elsif inner.respond_to?(#{lookup.inspect})"
expr.indent do
expr.line "inner.#{lookup}.to_liquid"
end
end
expr.line "end"
end
end
expr.line "end)"
expr.string.strip
end
private
def line(string)
output.line(string)
end
def echo(string)
unless @blank
line "liquid_out.write(to_output(#{string}))"
end
end
def indent(&block)
output.indent(&block)
end
def var(name)
name = name
.gsub('_', '__')
.gsub('-', '_')
"__liquid_#{name}"
end
def unvar(name)
name
.delete_prefix('__liquid_')
.gsub(/([^_])_([^_])/) { "#$1-#$2" }
.gsub('__', '_')
end
def hoist_var(name)
@variables << var(name)
end
attr_reader :output
end
end

View File

@@ -5,8 +5,8 @@ module Liquid
include ParserSwitching
class << self
def parse(tag_name, markup, tokenizer, options)
tag = new(tag_name, markup, options)
def parse(tag_name, markup, tokenizer, parse_context)
tag = new(tag_name, markup, parse_context)
tag.parse(tokenizer)
tag
end

View File

@@ -10,6 +10,10 @@ module Liquid
class Assign < Tag
Syntax = /(#{VariableSignature}+)\s*=\s*(.*)\s*/om
def self.syntax_error_translation_key
"errors.syntax.assign".freeze
end
attr_reader :to, :from
def initialize(tag_name, markup, options)
@@ -18,7 +22,7 @@ module Liquid
@to = $1
@from = Variable.new($2, options)
else
raise SyntaxError.new options[:locale].t("errors.syntax.assign".freeze)
raise SyntaxError.new(options[:locale].t(self.class.syntax_error_translation_key))
end
end

View File

@@ -13,8 +13,6 @@ module Liquid
class Capture < Block
Syntax = /(#{VariableSignature}+)/o
attr_reader :to
def initialize(tag_name, markup, options)
super
if markup =~ Syntax
@@ -35,7 +33,6 @@ module Liquid
def blank?
true
end
end
Template.register_tag('capture'.freeze, Capture)

View File

@@ -15,7 +15,7 @@ module Liquid
SimpleSyntax = /\A#{QuotedFragment}+/o
NamedSyntax = /\A(#{QuotedFragment})\s*\:\s*(.*)/om
attr_reader :variables, :name
attr_reader :variables
def initialize(tag_name, markup, options)
super

View File

@@ -23,8 +23,6 @@ module Liquid
@variable = markup.strip
end
attr_reader :variable
def render_to_output_buffer(context, output)
value = context.environments.first[@variable] ||= 0
value -= 1

24
lib/liquid/tags/echo.rb Normal file
View File

@@ -0,0 +1,24 @@
module Liquid
# Echo outputs an expression
#
# {% echo monkey %}
# {% echo user.name %}
#
# This is identical to variable output syntax, like {{ foo }}, but works
# inside {% liquid %} tags. The full syntax is supported, including filters:
#
# {% echo user | link %}
#
class Echo < Tag
def initialize(tag_name, markup, parse_context)
super
@variable = Variable.new(markup, parse_context)
end
def render(context)
@variable.render_to_output_buffer(context, '')
end
end
Template.register_tag('echo'.freeze, Echo)
end

View File

@@ -46,7 +46,7 @@ module Liquid
class For < Block
Syntax = /\A(#{VariableSegment}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o
attr_reader :collection_name, :variable_name, :limit, :from, :for_block, :else_block, :name, :reversed
attr_reader :collection_name, :variable_name, :limit, :from
def initialize(tag_name, markup, options)
super

View File

@@ -20,8 +20,6 @@ module Liquid
@variable = markup.strip
end
attr_reader :variable
def render_to_output_buffer(context, output)
value = context.environments.first[@variable] ||= 0
context.environments.first[@variable] = value + 1

View File

@@ -3,8 +3,6 @@ module Liquid
Syntax = /\A\s*\z/
FullTokenPossiblyInvalid = /\A(.*)#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om
attr_reader :body
def initialize(tag_name, markup, parse_context)
super

View File

@@ -206,7 +206,7 @@ module Liquid
begin
# render the nodelist.
# for performance reasons we get an array back here. join will make a string out of it.
with_profiling(context) do
@root.render_to_output_buffer(context, output || '')
end

View File

@@ -1,25 +1,31 @@
module Liquid
class Tokenizer
attr_reader :line_number
attr_reader :line_number, :for_liquid_tag
def initialize(source, line_numbers = false)
def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
@source = source
@line_number = line_numbers ? 1 : nil
@line_number = line_number || (line_numbers ? 1 : nil)
@for_liquid_tag = for_liquid_tag
@tokens = tokenize
end
def shift
token = @tokens.shift
@line_number += token.count("\n") if @line_number && token
token = @tokens.shift or return
if @line_number
@line_number += @for_liquid_tag ? 1 : token.count("\n")
end
token
end
private
def tokenize
@source = @source.source if @source.respond_to?(:source)
return [] if @source.to_s.empty?
return @source.split("\n") if @for_liquid_tag
tokens = @source.split(TemplateParser)
# removes the rogue empty element at the beginning of the array

View File

@@ -3,7 +3,7 @@ module Liquid
SQUARE_BRACKETED = /\A\[(.*)\]\z/m
COMMAND_METHODS = ['size'.freeze, 'first'.freeze, 'last'.freeze].freeze
attr_reader :name, :lookups, :command_flags
attr_reader :name, :lookups
def self.parse(markup)
new(markup)

View File

@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
s.license = "MIT"
# s.description = "A secure, non-evaling end user template engine with aesthetic markup."
s.required_ruby_version = ">= 2.1.0"
s.required_ruby_version = ">= 2.4.0"
s.required_rubygems_version = ">= 1.3.7"
s.test_files = Dir.glob("{test}/**/*")

View File

@@ -1,16 +1,7 @@
require 'benchmark/ips'
require_relative 'theme_runner'
case ARGV.first.to_sym
when :lax
Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
when :strict
Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
when :superfluid
require 'liquid/superfluid'
Liquid::Template.error_mode = :strict
end
Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
profiler = ThemeRunner.new
Benchmark.ips do |x|

View File

@@ -1,26 +1,63 @@
# frozen_string_literal: true
requirf 'benchmark/ips'
require 'benchmark/ips'
require 'memory_profiler'
require 'terminal-table'
require_relative 'theme_runner'
def profile(phase, &block)
puts
puts "#{phase}:"
puts
class Profiler
LOG_LABEL = "Profiling: ".rjust(14).freeze
REPORTS_DIR = File.expand_path('.memprof', __dir__).freeze
report = MemoryProfiler.report(&block)
def self.run
puts
yield new
end
report.pretty_print(
color_output: true,
scale_bytes: true,
detailed_report: true
)
def initialize
@allocated = []
@retained = []
@headings = []
end
def profile(phase, &block)
print LOG_LABEL
print "#{phase}.. ".ljust(10)
report = MemoryProfiler.report(&block)
puts 'Done.'
@headings << phase.capitalize
@allocated << "#{report.scale_bytes(report.total_allocated_memsize)} (#{report.total_allocated} objects)"
@retained << "#{report.scale_bytes(report.total_retained_memsize)} (#{report.total_retained} objects)"
return if ENV['CI']
require 'fileutils'
report_file = File.join(REPORTS_DIR, "#{sanitize(phase)}.txt")
FileUtils.mkdir_p(REPORTS_DIR)
report.pretty_print(to_file: report_file, scale_bytes: true)
end
def tabulate
table = Terminal::Table.new(headings: @headings.unshift('Phase')) do |t|
t << @allocated.unshift('Total allocated')
t << @retained.unshift('Total retained')
end
puts
puts table
puts "\nDetailed report(s) saved to #{REPORTS_DIR}/" unless ENV['CI']
end
def sanitize(string)
string.downcase.gsub(/[\W]/, '-').squeeze('-')
end
end
Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
profiler = ThemeRunner.new
profile("Parsing") { profiler.compile }
profile("Rendering") { profiler.render }
runner = ThemeRunner.new
Profiler.run do |x|
x.profile('parse') { runner.compile }
x.profile('render') { runner.render }
x.tabulate
end

View File

@@ -0,0 +1,11 @@
require 'test_helper'
class EchoTest < Minitest::Test
include Liquid
def test_echo_outputs_its_input
assert_template_result('BAR', <<~LIQUID, { 'variable-name' => 'bar' })
{%- echo variable-name | upcase -%}
LIQUID
end
end

View File

@@ -0,0 +1,104 @@
require 'test_helper'
class LiquidTagTest < Minitest::Test
include Liquid
def test_liquid_tag
assert_template_result('1 2 3', <<~LIQUID, 'array' => [1, 2, 3])
{%- liquid
echo array | join: " "
-%}
LIQUID
assert_template_result('1 2 3', <<~LIQUID, 'array' => [1, 2, 3])
{%- liquid
for value in array
echo value
unless forloop.last
echo " "
endunless
endfor
-%}
LIQUID
assert_template_result('4 8 12 6', <<~LIQUID, 'array' => [1, 2, 3])
{%- liquid
for value in array
assign double_value = value | times: 2
echo double_value | times: 2
unless forloop.last
echo " "
endunless
endfor
echo " "
echo double_value
-%}
LIQUID
assert_template_result('abc', <<~LIQUID)
{%- liquid echo "a" -%}
b
{%- liquid echo "c" -%}
LIQUID
end
def test_liquid_tag_errors
assert_match_syntax_error("syntax error (line 1): Unknown tag 'error'", <<~LIQUID)
{%- liquid error no such tag -%}
LIQUID
assert_match_syntax_error("syntax error (line 7): Unknown tag 'error'", <<~LIQUID)
{{ test }}
{%-
liquid
for value in array
error no such tag
endfor
-%}
LIQUID
assert_match_syntax_error("syntax error (line 2): Unknown tag '!!! the guards are vigilant'", <<~LIQUID)
{%- liquid
!!! the guards are vigilant
-%}
LIQUID
assert_match_syntax_error("syntax error (line 4): 'for' tag was never closed", <<~LIQUID)
{%- liquid
for value in array
echo 'forgot to close the for tag'
-%}
LIQUID
end
def test_line_number_is_correct_after_a_blank_token
assert_match_syntax_error("syntax error (line 3): Unknown tag 'error'", "{% liquid echo ''\n\n error %}")
assert_match_syntax_error("syntax error (line 3): Unknown tag 'error'", "{% liquid echo ''\n \n error %}")
end
def test_cannot_open_blocks_living_past_a_liquid_tag
assert_match_syntax_error("syntax error (line 3): 'if' tag was never closed", <<~LIQUID)
{%- liquid
if true
-%}
{%- endif -%}
LIQUID
end
def test_quirk_can_close_blocks_created_before_a_liquid_tag
assert_template_result("42", <<~LIQUID)
{%- if true -%}
42
{%- liquid endif -%}
LIQUID
end
def test_liquid_tag_in_raw
assert_template_result("{% liquid echo 'test' %}\n", <<~LIQUID)
{% raw %}{% liquid echo 'test' %}{% endraw %}
LIQUID
end
end

View File

@@ -14,16 +14,11 @@ if env_mode = ENV['LIQUID_PARSER_MODE']
end
Liquid::Template.error_mode = mode
if ENV['LIQUID-C'] == '1'
if ENV['LIQUID_C'] == '1'
puts "-- LIQUID C"
require 'liquid/c'
end
if ENV['SUPERFLUID'] == '1'
puts "-- SUPERFLUID"
require 'liquid/superfluid'
end
if Minitest.const_defined?('Test')
# We're on Minitest 5+. Nothing to do here.
else
@@ -42,18 +37,18 @@ module Minitest
include Liquid
def assert_template_result(expected, template, assigns = {}, message = nil)
assert_equal expected, Template.parse(template).render!(assigns), message
assert_equal expected, Template.parse(template, line_numbers: true).render!(assigns), message
end
def assert_template_result_matches(expected, template, assigns = {}, message = nil)
return assert_template_result(expected, template, assigns, message) unless expected.is_a? Regexp
assert_match expected, Template.parse(template).render!(assigns), message
assert_match expected, Template.parse(template, line_numbers: true).render!(assigns), message
end
def assert_match_syntax_error(match, template, assigns = {})
exception = assert_raises(Liquid::SyntaxError) do
Template.parse(template).render(assigns)
Template.parse(template, line_numbers: true).render(assigns)
end
assert_match match, exception.message
end