From a179fd84a30d7b1e1a3f682bf88bd1bf21f6b0f2 Mon Sep 17 00:00:00 2001 From: SamDoiron Date: Tue, 21 Jan 2020 22:12:20 -0500 Subject: [PATCH] WIP --- Rakefile | 5 + lib/liquid/block_body.rb | 8 +- lib/liquid/condition.rb | 6 +- lib/liquid/context.rb | 10 + lib/liquid/superfluid.rb | 516 ++++++++++++++++++++++++++++------ lib/liquid/tags/cycle.rb | 2 +- lib/liquid/tags/decrement.rb | 2 + lib/liquid/tags/for.rb | 2 +- lib/liquid/tags/increment.rb | 2 + lib/liquid/tags/raw.rb | 2 + lib/liquid/template.rb | 2 +- lib/liquid/variable_lookup.rb | 2 +- performance/benchmark.rb | 11 +- performance/memory_profile.rb | 2 +- 14 files changed, 474 insertions(+), 98 deletions(-) diff --git a/Rakefile b/Rakefile index dc60e54..b156c8b 100755 --- a/Rakefile +++ b/Rakefile @@ -82,6 +82,11 @@ namespace :benchmark do task :strict do ruby "./performance/benchmark.rb strict" end + + desc "Run the liquid benchmark with strict parsing" + task :superfluid do + ruby "./performance/benchmark.rb superfluid" + end end namespace :profile do diff --git a/lib/liquid/block_body.rb b/lib/liquid/block_body.rb index c2478ce..43dadb6 100644 --- a/lib/liquid/block_body.rb +++ b/lib/liquid/block_body.rb @@ -97,7 +97,7 @@ module Liquid end idx += 1 - raise_if_resource_limits_reached(context, output.bytesize - previous_output_size) + context.raise_if_resource_limits_reached(output.bytesize - previous_output_size) end output @@ -114,12 +114,6 @@ module Liquid output << context.handle_error(e, line_number) end - def raise_if_resource_limits_reached(context, length) - context.resource_limits.render_length += length - return unless context.resource_limits.reached? - raise MemoryError.new("Memory limits exceeded".freeze) - end - def create_variable(token, parse_context) token.scan(ContentOfVariable) do |content| markup = content.first diff --git a/lib/liquid/condition.rb b/lib/liquid/condition.rb index 3b51682..f6c79fd 100644 --- a/lib/liquid/condition.rb +++ b/lib/liquid/condition.rb @@ -29,7 +29,7 @@ module Liquid @@operators end - attr_reader :attachment, :child_condition + attr_reader :attachment, :child_condition, :child_relation attr_accessor :left, :operator, :right def initialize(left = nil, operator = nil, right = nil) @@ -81,10 +81,6 @@ module Liquid "#" end - protected - - attr_reader :child_relation - private def equal_variables(left, right) diff --git a/lib/liquid/context.rb b/lib/liquid/context.rb index 2dcc6af..1b2e75e 100644 --- a/lib/liquid/context.rb +++ b/lib/liquid/context.rb @@ -37,6 +37,16 @@ module Liquid @global_filter = nil end + def raise_argument_error(message) + raise Liquid::ArgumentError, message + end + + def raise_if_resource_limits_reached(length) + resource_limits.render_length += length + return unless resource_limits.reached? + raise MemoryError.new("Memory limits exceeded".freeze) + end + def warnings @warnings ||= [] end diff --git a/lib/liquid/superfluid.rb b/lib/liquid/superfluid.rb index 364f53b..c827a46 100644 --- a/lib/liquid/superfluid.rb +++ b/lib/liquid/superfluid.rb @@ -1,25 +1,33 @@ require 'ap' + require 'pry' +require 'stackprof' + AwesomePrint.defaults = { raw: true } module Liquid - Liquid::BlockBody.class_eval do def render_to_output_buffer(context, output) - ruby = Compiler.compile(@nodelist) - # puts - # puts "--------------------------- GENERATED RUBY -" - # puts ruby - # puts "--------------------------- /GENERATED RUBY -" + ruby = Compiler.compile(@nodelist) + + if false + puts + puts "--------------------------- GENERATED RUBY -" + line_number = 1 + puts(ruby.lines.map do |line| + "#{line_number}\t#{line}".tap { line_number += 1} + end) + puts "--------------------------- /GENERATED RUBY -" + end instructions = RubyVM::InstructionSequence.compile(ruby) output_io = StringIO.new - instructions.eval.call(output_io, context) + instructions.eval.call(output_io, context, Condition) output << output_io.string end @@ -29,7 +37,7 @@ module Liquid end class Output - attr_reader :string + attr_reader :string, :indent_level def initialize(initial_indent) @string = ''.dup @@ -37,8 +45,16 @@ module Liquid @indent_str = " " * initial_indent * 2 end - def <<(line) - @string << @indent_str << line << "\n" + def line(string) + @string << @indent_str << string << "\n" + end + + def echo(string) + output << "liquid_out.write(to_output(#{string}))" + end + + def indent(&block) + output.indent(&block) end def indent @@ -60,7 +76,9 @@ module Liquid end def initialize + @variables = Set.new @output = Output.new(2) + @blank = false end def ruby @@ -71,34 +89,6 @@ module Liquid ].join("\n") end - def compile(node) - case node - when Liquid::Document, Liquid::BlockBody - node.nodelist.collect(&method(:compile)) - when Array - node.collect(&method(:compile)) - when Liquid::Variable - compile_variable(node) - when Liquid::For - compile_for(node) - when Liquid::If - compile_if(node) - when Liquid::Template - compile(node.root) - when Liquid::Assign - compile_assign(node) - when Liquid::Case - compile_case(node) - when Liquid::Capture - compile_capture(node) - when String - compile_echo_literal(node) - when Liquid::Break - line "break" - else - raise SuperfluidError, "Unknown node type #{node.inspect}" - end - end def header <<~RUBY @@ -107,13 +97,136 @@ module Liquid end end - class GlobalVariableLookup - def method_missing(method_name, *) - @context.find_variable(method_name.to_s.delete_prefix("__liquid_")) + class ForloopDrop + def initialize(name, length, parentloop) + @name = name + @length = length + @parentloop = parentloop + @index = 0 end - def run(liquid_out, context) + attr_accessor :parentloop + + def [](value) + case value + when "length" + @length + when "name" + @name + when "index" + @index + 1 + when "index0" + @index + when "rindex" + @length - @index + when "rindex0" + @length - @index - 1 + when "first" + @index == 0 + when "last" + @index == @length - 1 + when "parentloop" + @parentloop + end + end + + def to_liquid + self + end + + def key?(*) + true + end + + private + + def increment! + @index += 1 + end + end + + def slice_collection(collection, from, limit) + to = if limit.nil? + nil + else + limit + from + end + + if (from != 0 || !to.nil?) && collection.respond_to?(:load_slice) + collection.load_slice(from, to) + else + slice_collection_using_each(collection, from, to) + end + end + + def slice_collection_using_each(collection, from, to) + segments = [] + index = 0 + + if collection.is_a?(String) + return collection.empty? ? [] : [collection] + end + return [] unless collection.respond_to?(:each) + + collection.each do |item| + if to && to <= index + break + end + + if from <= index + segments << item + end + + index += 1 + end + + segments + end + + def apply_operator(left, operator, right) + if left.respond_to?(operator) && right.respond_to?(operator) && !left.is_a?(Hash) && !right.is_a?(Hash) + begin + left.send(operator, right) + rescue ::ArgumentError => e + raise @context.raise_argument_error(e.message) + end + end + end + + def contains?(left, right) + if left && right && left.respond_to?(:include?) + right = right.to_s if left.is_a?(String) + left.include?(right) + else + false + end + end + + class GlobalVariableLookup + def method_missing(method_name, *) + nil + end + + def to_output(value) + output = if value.is_a?(Array) + value.join + elsif value == nil + else + value.to_s + end + @context.apply_global_filter(output) + end + + def run(liquid_out, context, condition) + @condition = condition @context = context + @for_offsets = {} + @cycle_values = {} + @context.registers[:for_stack] = [] + @context.registers[:cycle] ||= {} + @if_changed_last = nil + @prev_output_size = 0 + #{hoisted_variables} RUBY end @@ -125,37 +238,176 @@ module Liquid RUBY end + def hoisted_variables + @variables.map do |variable| + normal_name = unvar(variable) + "#{variable} = @context.find_variable(#{normal_name.inspect}, raise_on_not_found: false)" + end.join("\n") + end + + def compile(node) + old_blank = @blank + @blank = if node.respond_to?(:blank?) + node.blank? + elsif !(node.is_a?(String) && node =~ /\A\s*\z/) + false + else + @blank + end + + case node + when Liquid::Document, Liquid::BlockBody + node.nodelist.collect(&method(:compile)) + when Array + node.collect(&method(:compile)) + when Liquid::Variable + compile_variable(node) + when Liquid::For + compile_for(node) + when Liquid::If + compile_if(node) + when Liquid::Ifchanged + compile_if_changed(node) + when Liquid::Template + compile(node.root) + when Liquid::Assign + compile_assign(node) + when Liquid::Case + compile_case(node) + when Liquid::Capture + compile_capture(node) + when String + compile_echo_literal(node) + when Liquid::Break + line "break" if @in_loop + when Liquid::Continue + line "next" if @in_loop + when Liquid::Cycle + compile_cycle(node) + when Liquid::Raw + compile_raw(node) + when Liquid::Increment + compile_increment(node) + when Liquid::Decrement + compile_decrement(node) + when Liquid::Comment + else + raise SuperfluidError, "Unknown node type #{node.inspect}" + end + + @blank = old_blank + end + def compile_for(node) variable_name = node.variable_name collection_name = node.collection_name - iter_target_expr = case collection_name - when Liquid::VariableLookup - var(collection_name.name) - when Range - collection_name - when Liquid::RangeLookup - start_expr = make_variable_expr(collection_name.start_obj) - end_expr = make_variable_expr(collection_name.end_obj) - "#{start_expr}..#{end_expr}" - else - derp "Weird iter!", collection_name - end - line "(#{iter_target_expr}).each do |#{var(variable_name)}|" - indent { compile(node.for_block) } + iter_target_expr = case collection_name + when Liquid::VariableLookup + make_variable_lookup_expr(collection_name) + when Range + collection_name + when Liquid::RangeLookup + start_expr = make_variable_expr(collection_name.start_obj) + end_expr = make_variable_expr(collection_name.end_obj) + line "start = #{start_expr}" + line "@context.raise_argument_error('bad value for range') unless start.respond_to?(:to_i)" + line "start = #{start_expr}.to_i" + + line "finish = #{end_expr}" + line "@context.raise_argument_error('bad value for range') unless finish.respond_to?(:to_i)" + line "finish = #{end_expr}.to_i" + + "start..finish" + when Liquid::Expression::MethodLiteral + '[]' + else + raise SuperfluidError, "Unknown iteration target: #{collection_name.inspect}" + end + + from_expr = if node.from == :continue + "@for_offsets['#{node.name}'].to_i" + elsif node.from + make_variable_expr(node.from) + else + '0' + end + + limit_expr = node.limit ? make_variable_expr(node.limit) : 'nil' + line "from = #{from_expr}" + line "limit = #{limit_expr}" + line "@context.raise_argument_error('invalid integer') unless from.is_a?(Integer)" + line "@context.raise_argument_error('invalid integer') unless !limit || limit.is_a?(Integer)" + + line "segment = slice_collection(#{iter_target_expr}, from, limit)" + line "segment.reverse!" if node.reversed + + forloop = var('forloop') + hoist_var('forloop') + line "#{forloop} = ForloopDrop.new('#{node.name}', segment.length, #{forloop})" + + line "if segment.any?" + indent do + line "segment.each do |#{var(variable_name)}|" + indent do + line "@context['forloop'] = #{forloop}" + + + old_in_loop = @in_loop + compile(node.for_block) + @in_loop = old_in_loop + + line "#{forloop}.send(:increment!)" + end + line "end" + end + line "else" + indent do + compile(node.else_block) if node.else_block + end line "end" + + line "@for_offsets['#{node.name}'] = from + segment.length" + line "#{forloop} = #{forloop}.parentloop" end def compile_if(node) - condition = node.blocks.first - line "if #{make_condition_expr(condition)}" + if_condition = node.blocks.first + line "if #{make_condition_expr(if_condition)}" + indent { if_condition.attachment.nodelist.each(&method(:compile)) } + + node.blocks.drop(1).each do |condition| + if condition.left != nil + line "elsif #{make_condition_expr(condition)}" + else + line "else" + end indent { condition.attachment.nodelist.each(&method(:compile)) } + end + line "end" end + def compile_if_changed(node) + line "if_changed = lambda do |; liquid_out|" + indent do + line "liquid_out = StringIO.new" + node.nodelist.each(&method(:compile)) + line "liquid_out.string" + end + line "end.call" + + line "if if_changed != @if_changed_last" + indent { echo "if_changed" } + line "end" + + line "@if_changed_last = if_changed" + end + def compile_capture(node) line "#{var(node.to)} = lambda do |; liquid_out|" + hoist_var(node.to) indent do line "liquid_out = StringIO.new" node.nodelist.each(&method(:compile)) @@ -165,13 +417,43 @@ module Liquid end def make_condition_expr(node) - make_variable_expr(node) - if node.operator - left = make_variable_expr(node.left) - right = make_variable_expr(node.right) - "#{left} #{node.operator} #{right}" + condition = make_sub_condition_expr(node) + if node.child_condition + "(#{condition} #{node.child_relation} #{make_condition_expr(node.child_condition)})" else - make_variable_expr(node.left) + condition + end + end + + def make_sub_condition_expr(node) + return make_variable_expr(node.left) unless node.operator + + operator = node.operator + operator = "!=" if operator == "<>" + + if operator == "==" + if node.left.is_a?(Liquid::Expression::MethodLiteral) && + node.right.is_a?(Liquid::Expression::MethodLiteral) + return "false" + elsif node.right.is_a?(Liquid::Expression::MethodLiteral) + target = make_variable_expr(node.left) + message = node.right.method_name.inspect + return "#{target}.respond_to?(#{message}) ? #{target}.send(#{message}) : nil" + elsif node.left.is_a?(Liquid::Expression::MethodLiteral) + target = make_variable_expr(node.right) + message = node.left.method_name.inspect + return "#{target}.respond_to?(#{message}) ? #{target}.send(#{message}) : nil" + end + end + + left = make_variable_expr(node.left) + right = make_variable_expr(node.right) + + case operator + when "contains" + "contains?(#{left}, #{right})" + else + "apply_operator(#{left}, #{operator.inspect}, #{right})" end end @@ -196,6 +478,36 @@ module Liquid line "end" end + def compile_cycle(node) + key = node.name + key = key.name if key.is_a?(Liquid::VariableLookup) + + line "key = #{key.inspect}" + line "iteration = context.registers[:cycle][key].to_i" + line "@cycle_values[key] ||= #{node.variables}" + + line "val = @cycle_values[key][iteration]" + echo 'val' + line "context.registers[:cycle][key] = (iteration + 1) % #{node.variables.size}" + end + + def compile_raw(node) + echo "#{node.body.inspect}" + end + + def compile_increment(node) + line "value = context.environments.first[#{node.variable.inspect}] ||= 0" + line "@context.environments.first[#{node.variable.inspect}] = value + 1" + echo "value" + end + + def compile_decrement(node) + line "value = context.environments.first[#{node.variable.inspect}] ||= 0" + line "value -= 1" + line "@context.environments.first[#{node.variable.inspect}] = value" + echo "value" + end + def compile_echo_literal(node) echo node.inspect end @@ -209,29 +521,25 @@ module Liquid when Liquid::Variable make_variable_expr(node.from) else - derp "Weird assign", node + raise SuperfluidError, "Unknown assignment `from`: #{node.from.inspect}" end line "#{var(node.to)} = #{from_expr}" + hoist_var(node.to) end def make_variable_expr(variable) case variable when Liquid::Variable base_expression = case variable.name - when Integer - variable.name - when String + when TrueClass, FalseClass, Numeric, String variable.name.inspect - when TrueClass, FalseClass - variable.name when Liquid::VariableLookup - result = var(variable.name.name) - variable.name.lookups.each do |lookup| - result << "[#{make_variable_expr(lookup)}]" - end - result + make_variable_lookup_expr(variable.name) + when NilClass, Liquid::Expression::MethodLiteral + 'nil' else + raise SuperfluidError, "Invalid variable name: #{variable.name.inspect}" derp "Bad var name", variable.name end variable.filters.inject(base_expression) do |inner, (filter_name, positional_args, keyword_args)| @@ -244,23 +552,57 @@ module Liquid .join(", ") + " }" end - "context.strainer.invoke(#{filter_name.inspect}, #{inner}, #{filter_args.join(", ")})" + "context.strainer.invoke(#{filter_name.inspect}, #{inner}, *[#{filter_args.join(", ")}])" end when Liquid::VariableLookup - var(variable.name) - else + make_variable_lookup_expr(variable) + when TrueClass, FalseClass, Numeric, String variable.inspect + when NilClass + 'nil' + else + raise SuperfluidError, "Unknown expression type: #{variable.inspect}" end end + def make_variable_lookup_expr(variable_lookup) + base_expr = var(variable_lookup.name) + hoist_var(variable_lookup.name) + return base_expr if variable_lookup.lookups.empty? + + expr = Output.new(output.indent_level) + expr.line "(begin" + expr.indent do + expr.line "inner = #{base_expr}" + variable_lookup.lookups.each_with_index do |lookup, i| + lookup_expr = lookup.inspect + expr.line "inner = if inner.respond_to?(:[]) && ((inner.respond_to?(:key?) && inner.key?(#{lookup_expr})) || (inner.respond_to?(:fetch) && #{lookup_expr}.is_a?(Integer)))" + expr.indent do + expr.line "inner[#{lookup_expr}].to_liquid" + end + if variable_lookup.command_flags & (1 << i) != 0 + expr.line "elsif inner.respond_to?(#{lookup.inspect})" + expr.indent do + expr.line "inner.#{lookup}.to_liquid" + end + end + expr.line "end" + end + end + expr.line "end)" + expr.string.strip + end + private def line(string) - output << string + output.line(string) end def echo(string) - output << "liquid_out.write(#{string})" + unless @blank + line "liquid_out.write(to_output(#{string}))" + end end def indent(&block) @@ -268,9 +610,23 @@ module Liquid end def var(name) + name = name + .gsub('_', '__') + .gsub('-', '_') "__liquid_#{name}" end + def unvar(name) + name + .delete_prefix('__liquid_') + .gsub(/([^_])_([^_])/) { "#$1-#$2" } + .gsub('__', '_') + end + + def hoist_var(name) + @variables << var(name) + end + attr_reader :output end end diff --git a/lib/liquid/tags/cycle.rb b/lib/liquid/tags/cycle.rb index e42244d..d3a395e 100644 --- a/lib/liquid/tags/cycle.rb +++ b/lib/liquid/tags/cycle.rb @@ -15,7 +15,7 @@ module Liquid SimpleSyntax = /\A#{QuotedFragment}+/o NamedSyntax = /\A(#{QuotedFragment})\s*\:\s*(.*)/om - attr_reader :variables + attr_reader :variables, :name def initialize(tag_name, markup, options) super diff --git a/lib/liquid/tags/decrement.rb b/lib/liquid/tags/decrement.rb index 08ddd4d..b584e5f 100644 --- a/lib/liquid/tags/decrement.rb +++ b/lib/liquid/tags/decrement.rb @@ -23,6 +23,8 @@ module Liquid @variable = markup.strip end + attr_reader :variable + def render_to_output_buffer(context, output) value = context.environments.first[@variable] ||= 0 value -= 1 diff --git a/lib/liquid/tags/for.rb b/lib/liquid/tags/for.rb index a47002b..8d04d79 100644 --- a/lib/liquid/tags/for.rb +++ b/lib/liquid/tags/for.rb @@ -46,7 +46,7 @@ module Liquid class For < Block Syntax = /\A(#{VariableSegment}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o - attr_reader :collection_name, :variable_name, :limit, :from, :for_block + attr_reader :collection_name, :variable_name, :limit, :from, :for_block, :else_block, :name, :reversed def initialize(tag_name, markup, options) super diff --git a/lib/liquid/tags/increment.rb b/lib/liquid/tags/increment.rb index 5af1242..ba982eb 100644 --- a/lib/liquid/tags/increment.rb +++ b/lib/liquid/tags/increment.rb @@ -20,6 +20,8 @@ module Liquid @variable = markup.strip end + attr_reader :variable + def render_to_output_buffer(context, output) value = context.environments.first[@variable] ||= 0 context.environments.first[@variable] = value + 1 diff --git a/lib/liquid/tags/raw.rb b/lib/liquid/tags/raw.rb index 4fa75d9..f6c5320 100644 --- a/lib/liquid/tags/raw.rb +++ b/lib/liquid/tags/raw.rb @@ -3,6 +3,8 @@ module Liquid Syntax = /\A\s*\z/ FullTokenPossiblyInvalid = /\A(.*)#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om + attr_reader :body + def initialize(tag_name, markup, parse_context) super diff --git a/lib/liquid/template.rb b/lib/liquid/template.rb index 91e30fb..48ffb5b 100644 --- a/lib/liquid/template.rb +++ b/lib/liquid/template.rb @@ -206,7 +206,7 @@ module Liquid begin # render the nodelist. - # for performance reasons we get an array back here. join will make a string out of it. + with_profiling(context) do @root.render_to_output_buffer(context, output || '') end diff --git a/lib/liquid/variable_lookup.rb b/lib/liquid/variable_lookup.rb index 62f4877..77efbc1 100644 --- a/lib/liquid/variable_lookup.rb +++ b/lib/liquid/variable_lookup.rb @@ -3,7 +3,7 @@ module Liquid SQUARE_BRACKETED = /\A\[(.*)\]\z/m COMMAND_METHODS = ['size'.freeze, 'first'.freeze, 'last'.freeze].freeze - attr_reader :name, :lookups + attr_reader :name, :lookups, :command_flags def self.parse(markup) new(markup) diff --git a/performance/benchmark.rb b/performance/benchmark.rb index 68c568c..5e669aa 100644 --- a/performance/benchmark.rb +++ b/performance/benchmark.rb @@ -1,7 +1,16 @@ require 'benchmark/ips' require_relative 'theme_runner' -Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first +case ARGV.first.to_sym +when :lax + Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first +when :strict + Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first +when :superfluid + require 'liquid/superfluid' + Liquid::Template.error_mode = :strict +end + profiler = ThemeRunner.new Benchmark.ips do |x| diff --git a/performance/memory_profile.rb b/performance/memory_profile.rb index bfacde8..269dcb0 100644 --- a/performance/memory_profile.rb +++ b/performance/memory_profile.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'benchmark/ips' +requirf 'benchmark/ips' require 'memory_profiler' require_relative 'theme_runner'