Run liquidtruffle from the performance runner

Don't attempt to install stackprof gem on JRuby

.rubocop.yml

@@ -1,4 +1,6 @@
-inherit_from: ./.rubocop_todo.yml
+inherit_from:
+  - .rubocop_todo.yml
+  - ./.rubocop_todo.yml
 
 AllCops:
   Exclude:
@@ -29,7 +31,7 @@ Lint/ParenthesesAsGroupedExpression:
 Lint/UnusedBlockArgument:
   Enabled: false
 
-Lint/EndAlignment:
+Layout/EndAlignment:
   EnforcedStyleAlignWith: variable
 
 Lint/UnusedMethodArgument:
@@ -59,7 +61,7 @@ Style/BracesAroundHashParameters:
 Style/NumericLiterals:
   Enabled: false
 
-Layout/SpaceInsideBrackets:
+Layout/SpaceInsideArrayLiteralBrackets:
   Enabled: false
 
 Layout/SpaceBeforeBlockBraces:
@@ -71,7 +73,10 @@ Style/Documentation:
 Style/ClassAndModuleChildren:
   Enabled: false
 
-Style/TrailingCommaInLiteral:
+Style/TrailingCommaInArrayLiteral:
+  Enabled: false
+
+Style/TrailingCommaInHashLiteral:
   Enabled: false
 
 Layout/IndentHash:
@@ -104,7 +109,7 @@ Style/SymbolLiteral:
 Performance/Count:
   Enabled: false
 
-Style/ConstantName:
+Naming/ConstantName:
   Enabled: false
 
 Layout/CaseIndentation:
@@ -122,6 +127,6 @@ Style/TrivialAccessors:
 Style/WordArray:
   Enabled: false
 
-Style/MethodName:
+Naming/MethodName:
   Exclude:
     - 'example/server/liquid_servlet.rb'

.rubocop_todo.yml

@@ -1,14 +1,22 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2019-02-03 21:12:39 +0530 using RuboCop version 0.49.1.
+# on 2019-03-19 11:04:37 -0400 using RuboCop version 0.53.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: Include, TreatCommentsAsGroupSeparators.
+# Include: **/*.gemspec
+Gemspec/OrderedDependencies:
+  Exclude:
+    - 'liquid.gemspec'
+
 # Offense count: 5
 # Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
+# Configuration parameters: EnforcedStyle.
 # SupportedStyles: auto_detection, squiggly, active_support, powerpack, unindent
 Layout/IndentHeredoc:
   Exclude:
@@ -17,7 +25,7 @@ Layout/IndentHeredoc:
 
 # Offense count: 6
 # Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
+# Configuration parameters: EnforcedStyle.
 # SupportedStyles: symmetrical, new_line, same_line
 Layout/MultilineMethodCallBraceLayout:
   Exclude:
@@ -26,38 +34,65 @@ Layout/MultilineMethodCallBraceLayout:
 
 # Offense count: 2
 # Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
+# Configuration parameters: EnforcedStyle.
 # SupportedStyles: runtime_error, standard_error
 Lint/InheritException:
   Exclude:
     - 'lib/liquid/interrupts.rb'
 
-# Offense count: 51
+# Offense count: 1
+# Configuration parameters: CheckForMethodsWithNoSideEffects.
+Lint/Void:
+  Exclude:
+    - 'lib/liquid/parse_context.rb'
+
+# Offense count: 54
 Metrics/AbcSize:
   Max: 56
 
-# Offense count: 11
+# Offense count: 12
 Metrics/CyclomaticComplexity:
   Max: 12
 
-# Offense count: 639
-# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
-# URISchemes: http, https
-Metrics/LineLength:
-  Max: 294
-
-# Offense count: 108
+# Offense count: 112
 # Configuration parameters: CountComments.
 Metrics/MethodLength:
   Max: 37
 
-# Offense count: 7
+# Offense count: 8
 Metrics/PerceivedComplexity:
   Max: 11
 
+# Offense count: 52
+# Configuration parameters: Blacklist.
+# Blacklist: END, (?-mix:EO[A-Z]{1})
+Naming/HeredocDelimiterNaming:
+  Exclude:
+    - 'test/integration/assign_test.rb'
+    - 'test/integration/capture_test.rb'
+    - 'test/integration/trim_mode_test.rb'
+
+# Offense count: 23
+# Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames.
+# AllowedNames: io, id
+Naming/UncommunicativeMethodParamName:
+  Exclude:
+    - 'example/server/example_servlet.rb'
+    - 'lib/liquid/condition.rb'
+    - 'lib/liquid/context.rb'
+    - 'lib/liquid/standardfilters.rb'
+    - 'lib/liquid/tags/if.rb'
+    - 'lib/liquid/utils.rb'
+    - 'lib/liquid/variable.rb'
+    - 'test/integration/filter_test.rb'
+    - 'test/integration/standard_filter_test.rb'
+    - 'test/integration/tags/for_tag_test.rb'
+    - 'test/integration/template_test.rb'
+    - 'test/unit/condition_unit_test.rb'
+
 # Offense count: 10
 # Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
+# Configuration parameters: EnforcedStyle.
 # SupportedStyles: prefer_alias, prefer_alias_method
 Style/Alias:
   Exclude:
@@ -69,14 +104,23 @@ Style/Alias:
     - 'lib/liquid/tags/include.rb'
     - 'lib/liquid/variable.rb'
 
+# Offense count: 22
+Style/CommentedKeyword:
+  Enabled: false
+
 # Offense count: 1
 # Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles, SingleLineConditionsOnly, IncludeTernaryExpressions.
+# Configuration parameters: EnforcedStyle, SingleLineConditionsOnly, IncludeTernaryExpressions.
 # SupportedStyles: assign_to_condition, assign_inside_condition
 Style/ConditionalAssignment:
   Exclude:
     - 'lib/liquid/errors.rb'
 
+# Offense count: 1
+Style/DateTime:
+  Exclude:
+    - 'test/unit/context_unit_test.rb'
+
 # Offense count: 2
 # Cop supports --auto-correct.
 Style/EmptyCaseCondition:
@@ -86,7 +130,7 @@ Style/EmptyCaseCondition:
 
 # Offense count: 5
 # Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles.
+# Configuration parameters: EnforcedStyle.
 # SupportedStyles: compact, expanded
 Style/EmptyMethod:
   Exclude:
@@ -96,11 +140,28 @@ Style/EmptyMethod:
     - 'test/integration/tags/include_tag_test.rb'
     - 'test/unit/context_unit_test.rb'
 
+# Offense count: 3
+# Cop supports --auto-correct.
+Style/Encoding:
+  Exclude:
+    - 'lib/liquid/version.rb'
+    - 'liquid.gemspec'
+    - 'test/integration/standard_filter_test.rb'
+
 # Offense count: 2
-# Configuration parameters: SupportedStyles.
-# SupportedStyles: annotated, template
+# Cop supports --auto-correct.
+Style/ExpandPathArguments:
+  Exclude:
+    - 'Rakefile'
+    - 'liquid.gemspec'
+
+# Offense count: 7
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: annotated, template, unannotated
 Style/FormatStringToken:
-  EnforcedStyle: template
+  Exclude:
+    - 'test/integration/filter_test.rb'
+    - 'test/integration/hash_ordering_test.rb'
 
 # Offense count: 14
 # Configuration parameters: MinBodyLength.
@@ -121,7 +182,7 @@ Style/GuardClause:
 
 # Offense count: 1
 # Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, MinBodyLength, SupportedStyles.
+# Configuration parameters: EnforcedStyle, MinBodyLength.
 # SupportedStyles: skip_modifier_ifs, always
 Style/Next:
   Exclude:
@@ -129,7 +190,7 @@ Style/Next:
 
 # Offense count: 4
 # Cop supports --auto-correct.
-# Configuration parameters: AutoCorrect, EnforcedStyle, SupportedStyles.
+# Configuration parameters: AutoCorrect, EnforcedStyle.
 # SupportedStyles: predicate, comparison
 Style/NumericPredicate:
   Exclude:
@@ -166,14 +227,14 @@ Style/Semicolon:
 
 # Offense count: 7
 # Cop supports --auto-correct.
-# Configuration parameters: MinSize, SupportedStyles.
+# Configuration parameters: MinSize.
 # SupportedStyles: percent, brackets
 Style/SymbolArray:
   EnforcedStyle: brackets
 
 # Offense count: 2
 # Cop supports --auto-correct.
-# Configuration parameters: EnforcedStyle, SupportedStyles, AllowSafeAssignment.
+# Configuration parameters: EnforcedStyle, AllowSafeAssignment.
 # SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex
 Style/TernaryParentheses:
   Exclude:
@@ -188,7 +249,12 @@ Style/UnneededPercentQ:
 
 # Offense count: 1
 # Cop supports --auto-correct.
-# Configuration parameters: MaxLineLength.
 Style/WhileUntilModifier:
   Exclude:
     - 'lib/liquid/tags/case.rb'
+
+# Offense count: 640
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 294

.travis.yml

@@ -23,7 +23,6 @@ matrix:
     - rvm: jruby-head
 
 install:
-  - gem install rainbow -v 2.2.1
   - bundle install
 
 script: bundle exec rake

Gemfile

@@ -5,16 +5,23 @@ end
 
 gemspec
 
-gem 'stackprof', platforms: :mri
-
 group :benchmark, :test do
   gem 'benchmark-ips'
+  gem 'memory_profiler'
+
+  # install_if -> { RUBY_PLATFORM !~ /mingw|mswin|java/ } do
+  #   gem 'stackprof'
+  # end
 end
 
 group :test do
-  gem 'rubocop', '~> 0.49.0'
+  gem 'rubocop', '~> 0.53.0'
 
   platform :mri do
     gem 'liquid-c', github: 'Shopify/liquid-c', ref: '9168659de45d6d576fce30c735f857e597fa26f6'
   end
 end
+
+# gem "byebug", "~> 11.0"
+
+gem "pry", "~> 0.12.2"

Rakefile

@@ -85,6 +85,13 @@ namespace :profile do
   end
 end
 
+namespace :memory_profile do
+  desc "Run memory profiler"
+  task :run do
+    ruby "./performance/memory_profile.rb"
+  end
+end
+
 desc "Run example"
 task :example do
   ruby "-w -d -Ilib example/server/server.rb"
@@ -93,3 +100,9 @@ end
 task :console do
   exec 'irb -I lib -r liquid'
 end
+
+task :dump_portable_ast, [:filename] do |_task, args|
+  require 'liquid/ast_to_portable_json'
+  ast = Liquid::Template.parse(File.read(args[:filename]))
+  puts(Liquid::ASTToPortableJSON.dump(ast))
+end

lib/liquid/ast_to_portable_json.rb

@@ -0,0 +1,27 @@
+require 'yaml'
+require 'json'
+require 'liquid'
+
+module Liquid
+  class ASTToPortableJSON
+    def self.dump(ast)
+      yaml = YAML.dump(ast)
+      yaml.gsub!(/---.*/, '')
+      yaml.gsub!(/!ruby\/object:(.+)\n(\s+)/, "\n\\2class_name: \\1\n\\2")
+
+      bare_hash_ast = YAML.load(yaml)
+      delete_parse_context(bare_hash_ast)
+      JSON.pretty_generate(bare_hash_ast)
+    end
+
+    def self.delete_parse_context(bare_hash_ast)
+      if bare_hash_ast.is_a?(Array)
+        bare_hash_ast.each { |v| delete_parse_context(v) }
+      elsif bare_hash_ast.is_a?(Hash)
+        bare_hash_ast.delete('parse_context')
+        bare_hash_ast.each { |k, v| delete_parse_context(v) }
+      end
+    end
+  end
+end
+

lib/liquid/block_body.rb

@@ -115,7 +115,7 @@ module Liquid
     end
 
     def check_resources(context, node_output)
-      context.resource_limits.render_length += node_output.length
+      context.resource_limits.render_length += node_output.bytesize
       return unless context.resource_limits.reached?
       raise MemoryError.new("Memory limits exceeded".freeze)
     end

lib/liquid/tags/assign.rb

@@ -37,7 +37,7 @@ module Liquid
 
     def assign_score_of(val)
       if val.instance_of?(String)
-        val.length
+        val.bytesize
       elsif val.instance_of?(Array) || val.instance_of?(Hash)
         sum = 1
         # Uses #each to avoid extra allocations.

lib/liquid/tags/capture.rb

@@ -25,7 +25,7 @@ module Liquid
     def render(context)
       output = super
       context.scopes.last[@to] = output
-      context.resource_limits.assign_score += output.length
+      context.resource_limits.assign_score += output.bytesize
       ''.freeze
     end
 

lib/liquid/tags/for.rb

@@ -124,14 +124,23 @@ module Liquid
       from = if @from == :continue
         offsets[@name].to_i
       else
-        context.evaluate(@from).to_i
+        from_value = context.evaluate(@from)
+        if from_value.nil?
+          0
+        else
+          Utils.to_integer(from_value)
+        end
       end
 
       collection = context.evaluate(@collection_name)
       collection = collection.to_a if collection.is_a?(Range)
 
-      limit = context.evaluate(@limit)
-      to = limit ? limit.to_i + from : nil
+      limit_value = context.evaluate(@limit)
+      to = if limit_value.nil?
+        nil
+      else
+        Utils.to_integer(limit_value) + from
+      end
 
       segment = Utils.slice_collection(collection, from, to)
       segment.reverse! if @reversed

lib/liquid/template.rb

@@ -1,3 +1,4 @@
+require 'liquid/ast_to_portable_json'
 module Liquid
   # Templates are central to liquid.
   # Interpretating templates is a two step process. First you compile the
@@ -130,6 +131,8 @@ module Liquid
       @line_numbers = options[:line_numbers] || @profiling
       parse_context = options.is_a?(ParseContext) ? options : ParseContext.new(options)
       @root = Document.parse(tokenize(source), parse_context)
+      @json = ASTToPortableJSON.dump(self)
+
       @warnings = parse_context.warnings
       self
     end
@@ -198,6 +201,15 @@ module Liquid
         context.add_filters(args.pop)
       end
 
+      unless @truffle_context
+        environment = context.environments.first
+        @truffle_context = Polyglot.eval('liquid', @json)
+        environment.each do |key, value|
+          @truffle_context[key] = value
+        end
+      end
+      return @truffle_context[:render].call
+
       # Retrying a render resets resource usage
       context.resource_limits.reset
 

lib/liquid/variable.rb

@@ -95,16 +95,17 @@ module Liquid
 
     def parse_filter_expressions(filter_name, unparsed_args)
       filter_args = []
-      keyword_args = {}
+      keyword_args = nil
       unparsed_args.each do |a|
         if matches = a.match(JustTagAttributes)
+          keyword_args ||= {}
           keyword_args[matches[1]] = Expression.parse(matches[2])
         else
           filter_args << Expression.parse(a)
         end
       end
       result = [filter_name, filter_args]
-      result << keyword_args unless keyword_args.empty?
+      result << keyword_args if keyword_args
       result
     end
 

performance/benchmark.rb

@@ -1,3 +1,4 @@
+require 'bundler/setup'
 require 'benchmark/ips'
 require_relative 'theme_runner'
 
@@ -5,14 +6,15 @@ Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
 profiler = ThemeRunner.new
 
 Benchmark.ips do |x|
-  x.time = 10
-  x.warmup = 5
+  x.time = 60
+  x.warmup = 1
 
   puts
   puts "Running benchmark for #{x.time} seconds (with #{x.warmup} seconds warmup)."
   puts
 
-  x.report("parse:") { profiler.compile }
+  profiler.compile
+  # x.report("parse:") { profiler.compile }
   x.report("render:") { profiler.render }
   x.report("parse & render:") { profiler.run }
 end

performance/memory_profile.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'benchmark/ips'
+require 'memory_profiler'
+require_relative 'theme_runner'
+
+def profile(phase, &block)
+  puts
+  puts "#{phase}:"
+  puts
+
+  report = MemoryProfiler.report(&block)
+
+  report.pretty_print(
+    color_output: true,
+    scale_bytes: true,
+    detailed_report: true
+  )
+end
+
+Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
+
+profiler = ThemeRunner.new
+
+profile("Parsing") { profiler.compile }
+profile("Rendering") { profiler.render }

performance/theme_runner.rb

@@ -24,7 +24,7 @@ class ThemeRunner
   # Initialize a new liquid ThemeRunner instance
   # Will load all templates into memory, do this now so that we don't profile IO.
   def initialize
-    @tests = Dir[__dir__ + '/tests/**/*.liquid'].collect do |test|
+    @tests = Dir[__dir__ + '/tests/ripen/product.liquid'].collect do |test|
       next if File.basename(test) == 'theme.liquid'
 
       theme_path = File.dirname(test) + '/theme.liquid'

test/integration/tags/for_tag_test.rb

@@ -103,6 +103,34 @@ HERE
     assert_template_result('3456', '{%for i in array limit: 4 offset: 2 %}{{ i }}{%endfor%}', assigns)
   end
 
+  def test_limiting_with_invalid_limit
+    assigns = { 'array' => [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] }
+    template = <<-MKUP
+      {% for i in array limit: true offset: 1 %}
+        {{ i }}
+      {% endfor %}
+    MKUP
+
+    exception = assert_raises(Liquid::ArgumentError) do
+      Template.parse(template).render!(assigns)
+    end
+    assert_equal("Liquid error: invalid integer", exception.message)
+  end
+
+  def test_limiting_with_invalid_offset
+    assigns = { 'array' => [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] }
+    template = <<-MKUP
+      {% for i in array limit: 1 offset: true %}
+        {{ i }}
+      {% endfor %}
+    MKUP
+
+    exception = assert_raises(Liquid::ArgumentError) do
+      Template.parse(template).render!(assigns)
+    end
+    assert_equal("Liquid error: invalid integer", exception.message)
+  end
+
   def test_dynamic_variable_limiting
     assigns = { 'array' => [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] }
     assigns['limit'] = 2

test/integration/template_test.rb

@@ -139,6 +139,16 @@ class TemplateTest < Minitest::Test
     refute_nil t.resource_limits.assign_score
   end
 
+  def test_resource_limits_assign_score_counts_bytes_not_characters
+    t = Template.parse("{% assign foo = 'すごい' %}")
+    t.render
+    assert_equal 9, t.resource_limits.assign_score
+
+    t = Template.parse("{% capture foo %}すごい{% endcapture %}")
+    t.render
+    assert_equal 9, t.resource_limits.assign_score
+  end
+
   def test_resource_limits_assign_score_nested
     t = Template.parse("{% assign foo = 'aaaa' | reverse %}")
 
@@ -187,6 +197,14 @@ class TemplateTest < Minitest::Test
     assert_equal "ababab", t.render
   end
 
+  def test_render_length_uses_number_of_bytes_not_characters
+    t = Template.parse("{% if true %}すごい{% endif %}")
+    t.resource_limits.render_length_limit = 10
+    assert_equal "Liquid error: Memory limits exceeded", t.render
+    t.resource_limits.render_length_limit = 18
+    assert_equal "すごい", t.render
+  end
+
   def test_default_resource_limits_unaffected_by_render_with_context
     context = Context.new
     t = Template.parse("{% for a in (1..100) %} {% assign foo = 1 %} {% endfor %}")