update grammar to support #

Refactor render_layout method for serialization

lib/liquid/block_body.rb

@@ -4,7 +4,7 @@ require 'English'
 
 module Liquid
   class BlockBody
-    LiquidTagToken      = /\A\s*(\w+)\s*(.*?)\z/o
+    LiquidTagToken      = /\A\s*([\w#]+)\s*(.*?)\z/o
     FullToken           = /\A#{TagStart}#{WhitespaceControl}?(\s*)(\w+)(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
     ContentOfVariable   = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
     WhitespaceOrNothing = /\A\s*\z/
@@ -99,7 +99,9 @@ module Liquid
     end
 
     private def parse_liquid_tag(markup, parse_context)
-      liquid_tag_tokenizer = Tokenizer.new(markup, line_number: parse_context.line_number, for_liquid_tag: true)
+      liquid_tag_tokenizer = parse_context.new_tokenizer(
+        markup, start_line_number: parse_context.line_number, for_liquid_tag: true
+      )
       parse_for_liquid_tag(liquid_tag_tokenizer, parse_context) do |end_tag_name, _end_tag_markup|
         if end_tag_name
           BlockBody.unknown_tag_in_liquid_tag(end_tag_name, parse_context)

lib/liquid/parse_context.rb

@@ -23,6 +23,10 @@ module Liquid
       Liquid::BlockBody.new
     end
 
+    def new_tokenizer(markup, start_line_number: nil, for_liquid_tag: false)
+      Tokenizer.new(markup, line_number: start_line_number, for_liquid_tag: for_liquid_tag)
+    end
+
     def parse_expression(markup)
       Expression.parse(markup)
     end

lib/liquid/tags/inline_comment.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Liquid
+  class InlineComment < Tag
+    def blank?
+      true
+    end
+
+    def render_to_output_buffer(_context, _output)
+    end
+  end
+
+  Template.register_tag('--', InlineComment)
+end
+

lib/liquid/template.rb

@@ -107,7 +107,8 @@ module Liquid
     # Returns self for easy chaining
     def parse(source, options = {})
       parse_context = configure_options(options)
-      @root         = Document.parse(tokenize(source), parse_context)
+      tokenizer     = parse_context.new_tokenizer(source, start_line_number: @line_numbers && 1)
+      @root         = Document.parse(tokenizer, parse_context)
       self
     end
 
@@ -223,10 +224,6 @@ module Liquid
       parse_context
     end
 
-    def tokenize(source)
-      Tokenizer.new(source, @line_numbers)
-    end
-
     def apply_options_to_context(context, options)
       context.add_filters(options[:filters]) if options[:filters]
       context.global_filter      = options[:global_filter] if options[:global_filter]

performance/theme_runner.rb

@@ -73,10 +73,14 @@ class ThemeRunner
 
   private
 
+  def render_layout(template, layout, assigns)
+    assigns['content_for_layout'] = template.render!(assigns)
+    layout&.render!(assigns)
+  end
+
   def compile_and_render(template, layout, assigns, page_template, template_file)
-    compiled_test                 = compile_test(template, layout, assigns, page_template, template_file)
-    assigns['content_for_layout'] = compiled_test[:tmpl].render!(assigns)
-    compiled_test[:layout].render!(assigns) if layout
+    compiled_test = compile_test(template, layout, assigns, page_template, template_file)
+    render_layout(compiled_test[:tmpl], compiled_test[:layout], compiled_test[:assigns])
   end
 
   def compile_all_tests

test/integration/inline_comment_test.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class InlineCommentTest < Minitest::Test
+  include Liquid
+
+  def test_basic_usage
+    template_source = <<-END_TEMPLATE
+    foo{% # this is a comment %}bar
+    END_TEMPLATE
+    template        = Template.parse(template_source)
+    rendered        = template.render!
+    assert_equal("foobar", rendered.strip)
+  end
+end
+

test/unit/tokenizer_unit_test.rb

@@ -32,21 +32,26 @@ class TokenizerTest < Minitest::Test
 
   private
 
+  def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)
+    parse_context.new_tokenizer(source, start_line_number: start_line_number)
+  end
+
   def tokenize(source)
-    tokenizer = Liquid::Tokenizer.new(source)
+    tokenizer = new_tokenizer(source)
     tokens    = []
-    while (t = tokenizer.shift)
+    # shift is private in Liquid::C::Tokenizer, since it is only for unit testing
+    while (t = tokenizer.send(:shift))
       tokens << t
     end
     tokens
   end
 
   def tokenize_line_numbers(source)
-    tokenizer    = Liquid::Tokenizer.new(source, true)
+    tokenizer    = new_tokenizer(source, start_line_number: 1)
     line_numbers = []
     loop do
       line_number = tokenizer.line_number
-      if tokenizer.shift
+      if tokenizer.send(:shift)
         line_numbers << line_number
       else
         break