Build the tokenizer through the parse context for liquid-c

.github/workflows/liquid.yml

@@ -6,8 +6,9 @@ jobs:
     strategy:
       matrix:
         entry:
-          - { ruby: 2.5, allowed-failure: false } # minimum supported
-          - { ruby: 3.0, allowed-failure: false } # latest
+          - { ruby: 2.5, allowed-failure: false }
+          - { ruby: 2.6, allowed-failure: false }
+          - { ruby: 2.7, allowed-failure: false }
           - { ruby: ruby-head, allowed-failure: true }
     name: test (${{ matrix.entry.ruby }})
     steps:

.rubocop.yml

@@ -1,7 +1,5 @@
-inherit_gem:
-  rubocop-shopify: rubocop.yml
-
 inherit_from:
+  - 'https://shopify.github.io/ruby-style-guide/rubocop.yml'
   - .rubocop_todo.yml
 
 require: rubocop-performance

CONTRIBUTING.md

@@ -5,7 +5,6 @@
 * Bugfixes
 * Performance improvements
 * Features that are likely to be useful to the majority of Liquid users
-* Documentation updates that are concise and likely to be useful to the majority of Liquid users
 
 ## Things we won't merge
 
@@ -15,14 +14,12 @@
 * Features that can easily be implemented on top of Liquid (for example as a custom filter or custom filesystem)
 * Code that does not include tests
 * Code that breaks existing tests
-* Documentation changes that are verbose, incorrect or not important to most people (we want to keep it simple and easy to understand)
 
 ## Workflow
 
-* [Sign the CLA](https://cla.shopify.com/) if you haven't already
 * Fork the Liquid repository
 * Create a new branch in your fork
-  * For updating [Liquid documentation](https://shopify.github.io/liquid/), create it from `gh-pages` branch. (You can skip tests.)
 * If it makes sense, add tests for your code and/or run a performance benchmark
 * Make sure all tests pass (`bundle exec rake`)
 * Create a pull request
+

Gemfile

@@ -19,7 +19,6 @@ end
 
 group :test do
   gem 'rubocop', '~> 1.4', require: false
-  gem 'rubocop-shopify', '~> 1.0.7', require: false
   gem 'rubocop-performance', require: false
 
   platform :mri, :truffleruby do

History.md

@@ -1,20 +1,5 @@
 # Liquid Change Log
 
-## 5.0.2 (unreleased)
-
-### Fixes
-* Fix support for using a String subclass for the liquid source (#1421) [Dylan Thacker-Smith]
-
-## 5.0.1 / 2021-03-24
-
-### Fixes
-* Add ParseTreeVisitor to Echo tag (#1414) [CP Clermont]
-* Test with ruby 3.0 as the latest ruby version (#1398) [Dylan Thacker-Smith]
-* Handle carriage return in newlines_to_br (#1391) [Unending]
-
-### Performance Improvements
-* Use split limit in truncatewords (#1361) [Dylan Thacker-Smith]
-
 ## 5.0.0 / 2021-01-06
 
 ### Features

lib/liquid/standardfilters.rb

@@ -89,15 +89,13 @@ module Liquid
 
     def truncatewords(input, words = 15, truncate_string = "...")
       return if input.nil?
-      input = input.to_s
-      words = Utils.to_integer(words)
-      words = 1 if words <= 0
+      wordlist = input.to_s.split
+      words    = Utils.to_integer(words)
 
-      wordlist = input.split(" ", words + 1)
-      return input if wordlist.length <= words
+      l = words - 1
+      l = 0 if l < 0
 
-      wordlist.pop
-      wordlist.join(" ").concat(truncate_string.to_s)
+      wordlist.length > l ? wordlist[0..l].join(" ").concat(truncate_string.to_s) : input
     end
 
     # Split input string into an array of substrings separated by given pattern.
@@ -297,7 +295,7 @@ module Liquid
 
     # Add <br /> tags in front of all newlines in input string
     def newline_to_br(input)
-      input.to_s.gsub(/\r?\n/, "<br />\n")
+      input.to_s.gsub(/\n/, "<br />\n")
     end
 
     # Reformat a date using Ruby's core Time#strftime( string ) -> string

lib/liquid/tags/echo.rb

@@ -12,8 +12,6 @@ module Liquid
   #   {% echo user | link %}
   #
   class Echo < Tag
-    attr_reader :variable
-
     def initialize(tag_name, markup, parse_context)
       super
       @variable = Variable.new(markup, parse_context)
@@ -22,12 +20,6 @@ module Liquid
     def render(context)
       @variable.render_to_output_buffer(context, +'')
     end
-
-    class ParseTreeVisitor < Liquid::ParseTreeVisitor
-      def children
-        [@node.variable]
-      end
-    end
   end
 
   Template.register_tag('echo', Echo)

lib/liquid/tokenizer.rb

@@ -5,7 +5,7 @@ module Liquid
     attr_reader :line_number, :for_liquid_tag
 
     def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
-      @source         = source.to_s.to_str
+      @source         = source
       @line_number    = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
       @tokens         = tokenize
@@ -24,7 +24,7 @@ module Liquid
     private
 
     def tokenize
-      return [] if @source.empty?
+      return [] if @source.to_s.empty?
 
       return @source.split("\n") if @for_liquid_tag
 

lib/liquid/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Liquid
-  VERSION = "5.0.2.alpha"
+  VERSION = "5.0.0"
 end

performance/theme_runner.rb

@@ -73,14 +73,10 @@ class ThemeRunner
 
   private
 
-  def render_layout(template, layout, assigns)
-    assigns['content_for_layout'] = template.render!(assigns)
-    layout&.render!(assigns)
-  end
-
   def compile_and_render(template, layout, assigns, page_template, template_file)
-    compiled_test = compile_test(template, layout, assigns, page_template, template_file)
-    render_layout(compiled_test[:tmpl], compiled_test[:layout], compiled_test[:assigns])
+    compiled_test                 = compile_test(template, layout, assigns, page_template, template_file)
+    assigns['content_for_layout'] = compiled_test[:tmpl].render!(assigns)
+    compiled_test[:layout].render!(assigns) if layout
   end
 
   def compile_all_tests

test/integration/context_test.rb

@@ -461,7 +461,6 @@ class ContextTest < Minitest::Test
   end
 
   def test_interrupt_avoids_object_allocations
-    @context.interrupt? # ruby 3.0.0 allocates on the first call
     assert_no_object_allocations do
       @context.interrupt?
     end

test/integration/standard_filter_test.rb

@@ -171,13 +171,10 @@ class StandardFiltersTest < Minitest::Test
     assert_equal('one two three', @filters.truncatewords('one two three'))
     assert_equal(
       'Two small (13&#8221; x 5.5&#8221; x 10&#8221; high) baskets fit inside one large basket (13&#8221;...',
-      @filters.truncatewords('Two small (13&#8221; x 5.5&#8221; x 10&#8221; high) baskets fit inside one large basket (13&#8221; x 16&#8221; x 10.5&#8221; high) with cover.', 15)
+        @filters.truncatewords('Two small (13&#8221; x 5.5&#8221; x 10&#8221; high) baskets fit inside one large basket (13&#8221; x 16&#8221; x 10.5&#8221; high) with cover.', 15)
     )
     assert_equal("测试测试测试测试", @filters.truncatewords('测试测试测试测试', 5))
     assert_equal('one two1', @filters.truncatewords("one two three", 2, 1))
-    assert_equal('one two three...', @filters.truncatewords("one  two\tthree\nfour", 3))
-    assert_equal('one two...', @filters.truncatewords("one two three four", 2))
-    assert_equal('one...', @filters.truncatewords("one two three four", 0))
   end
 
   def test_strip_html
@@ -542,7 +539,6 @@ class StandardFiltersTest < Minitest::Test
 
   def test_newlines_to_br
     assert_template_result("a<br />\nb<br />\nc", "{{ source | newline_to_br }}", 'source' => "a\nb\nc")
-    assert_template_result("a<br />\nb<br />\nc", "{{ source | newline_to_br }}", 'source' => "a\r\nb\nc")
   end
 
   def test_plus

test/integration/tags/if_else_tag_test.rb

@@ -187,49 +187,4 @@ class IfElseTagTest < Minitest::Test
       assert_template_result(expected.to_s, tpl, assigns, assigns.to_s)
     end
   end
-
-  def test_operators_order
-    expected_groupings = [
-      ->(w, x, y, z) { (w && (x || (y && z))) },
-      ->(w, x, y, z) { (w || (x && (y || z))) },
-    ]
-
-    liquid_conditions = [
-      'w and x or y and z',
-      'w or x and y or z',
-    ]
-
-    conditions = (0..0b1111).map do |cond|
-      [
-        (cond & 0b1000) == 0 ? false : true,
-        (cond & 0b0100) == 0 ? false : true,
-        (cond & 0b0010) == 0 ? false : true,
-        (cond & 0b0001) == 0 ? false : true,
-      ]
-    end
-
-    expected_results = expected_groupings.map do |test|
-      conditions.map do |condition|
-        test.call(*condition)
-      end
-    end
-
-    liquid_results = liquid_conditions.map do |conditional_expression|
-      tpl = "{% if #{conditional_expression} %}true{% else %}false{% endif %}"
-      template = Liquid::Template.parse(tpl, line_numbers: true)
-      conditions.map do |condition|
-        w, x, y, z = condition
-        assigns = { 'w' => w, 'x' => x, 'y' => y, 'z' => z }
-        template.render!(assigns)
-      end.map { |r| true?(r) }
-    end
-
-    assert_equal(expected_results, liquid_results)
-  end
-
-  private
-
-  def true?(obj)
-    obj.to_s.casecmp('true').zero?
-  end
 end

test/integration/tags/include_tag_test.rb

@@ -96,12 +96,12 @@ class IncludeTagTest < Minitest::Test
 
   def test_include_tag_with_alias
     assert_template_result("Product: Draft 151cm ",
-      "{% include 'product_alias' with products[0] as product %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
+                           "{% include 'product_alias' with products[0] as product %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
   end
 
   def test_include_tag_for_alias
     assert_template_result("Product: Draft 151cm Product: Element 155cm ",
-      "{% include 'product_alias' for products as product %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
+                           "{% include 'product_alias' for products as product %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
   end
 
   def test_include_tag_with_default_name

test/integration/tags/render_tag_test.rb

@@ -151,7 +151,7 @@ class RenderTagTest < Minitest::Test
     )
 
     assert_template_result("Product: Draft 151cm ",
-      "{% render 'product' with products[0] %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
+                           "{% render 'product' with products[0] %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
   end
 
   def test_render_tag_with_alias
@@ -161,7 +161,7 @@ class RenderTagTest < Minitest::Test
     )
 
     assert_template_result("Product: Draft 151cm ",
-      "{% render 'product_alias' with products[0] as product %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
+                           "{% render 'product_alias' with products[0] as product %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
   end
 
   def test_render_tag_for_alias
@@ -171,7 +171,7 @@ class RenderTagTest < Minitest::Test
     )
 
     assert_template_result("Product: Draft 151cm Product: Element 155cm ",
-      "{% render 'product_alias' for products as product %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
+                           "{% render 'product_alias' for products as product %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
   end
 
   def test_render_tag_for
@@ -181,7 +181,7 @@ class RenderTagTest < Minitest::Test
     )
 
     assert_template_result("Product: Draft 151cm Product: Element 155cm ",
-      "{% render 'product' for products %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
+                           "{% render 'product' for products %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
   end
 
   def test_render_tag_forloop
@@ -190,7 +190,7 @@ class RenderTagTest < Minitest::Test
     )
 
     assert_template_result("Product: Draft 151cm first  index:1 Product: Element 155cm  last index:2 ",
-      "{% render 'product' for products %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
+                           "{% render 'product' for products %}", "products" => [{ 'title' => 'Draft 151cm' }, { 'title' => 'Element 155cm' }])
   end
 
   def test_render_tag_for_drop
@@ -199,7 +199,7 @@ class RenderTagTest < Minitest::Test
     )
 
     assert_template_result("123",
-      "{% render 'loop' for loop as value %}", "loop" => TestEnumerable.new)
+                           "{% render 'loop' for loop as value %}", "loop" => TestEnumerable.new)
   end
 
   def test_render_tag_with_drop
@@ -208,6 +208,6 @@ class RenderTagTest < Minitest::Test
     )
 
     assert_template_result("TestEnumerable",
-      "{% render 'loop' with loop as value %}", "loop" => TestEnumerable.new)
+                           "{% render 'loop' with loop as value %}", "loop" => TestEnumerable.new)
   end
 end

test/integration/template_test.rb

@@ -323,18 +323,4 @@ class TemplateTest < Minitest::Test
     result = t.render('x' => 1, 'y' => 5)
     assert_equal('12345', result)
   end
-
-  def test_source_string_subclass
-    string_subclass = Class.new(String) do
-      # E.g. ActiveSupport::SafeBuffer does this, so don't just rely on to_s to return a String
-      def to_s
-        self
-      end
-    end
-    source = string_subclass.new("{% assign x = 2 -%} x= {{- x }}")
-    assert_instance_of(string_subclass, source)
-    output = Template.parse(source).render!
-    assert_equal("x=2", output)
-    assert_instance_of(String, output)
-  end
 end

test/unit/parse_tree_visitor_test.rb

@@ -26,13 +26,6 @@ class ParseTreeVisitorTest < Minitest::Test
     )
   end
 
-  def test_echo
-    assert_equal(
-      ["test"],
-      visit(%({% echo test %}))
-    )
-  end
-
   def test_if_condition
     assert_equal(
       ["test"],