cache variable lookups

Liquid UTF-8 support

History.md

@@ -1,27 +1,5 @@
 # Liquid Version History
 
-## 2.5.5 / 2014-01-10 / branch "2-5-stable"
-
-Security fix, cherry-picked from master (4e14a65):
-* Don't call to_sym when creating conditions for security reasons, see #273 [Bouke van der Bijl, bouk]
-* Prevent arbitrary method invocation on condition objects, see #274 [Dylan Thacker-Smith, dylanahsmith]
-
-## 2.5.4 / 2013-11-11
-
-* Fix "can't convert Fixnum into String" for "replace", see #173, [wǒ_is神仙, jsw0528]
-
-## 2.5.3 / 2013-10-09
-
-* #232, #234, #237: Fix map filter bugs [Florian Weingarten, fw42]
-
-## 2.5.2 / 2013-09-03 / deleted
-
-Yanked from rubygems, as it contained too many changes that broke compatibility. Those changes will be on following major releases.
-
-## 2.5.1 / 2013-07-24
-
-* #230: Fix security issue with map filter, Use invoke_drop in map filter [Florian Weingarten, fw42]
-
 ## 2.5.0 / 2013-03-06
 
 * Prevent Object methods from being called on drops

lib/liquid.rb

@@ -20,14 +20,15 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 module Liquid
+  WordRegex                   = RUBY_VERSION < "1.9" ? '\w' : '[[:word:]]'
   FilterSeparator             = /\|/
   ArgumentSeparator           = ','
   FilterArgumentSeparator     = ':'
   VariableAttributeSeparator  = '.'
   TagStart                    = /\{\%/
   TagEnd                      = /\%\}/
-  VariableSignature           = /\(?[\w\-\.\[\]]\)?/
-  VariableSegment             = /[\w\-]/
+  VariableSignature           = /\(?[#{WordRegex}\-\.\[\]]\)?/o
+  VariableSegment             = /[#{WordRegex}\-]/o
   VariableStart               = /\{\{/
   VariableEnd                 = /\}\}/
   VariableIncompleteEnd       = /\}\}?/
@@ -38,7 +39,7 @@ module Liquid
   OtherFilterArgument         = /#{ArgumentSeparator}(?:#{StrictQuotedFragment})/o
   SpacelessFilter             = /^(?:'[^']+'|"[^"]+"|[^'"])*#{FilterSeparator}(?:#{StrictQuotedFragment})(?:#{FirstFilterArgument}(?:#{OtherFilterArgument})*)?/o
   Expression                  = /(?:#{QuotedFragment}(?:#{SpacelessFilter})*)/o
-  TagAttributes               = /(\w+)\s*\:\s*(#{QuotedFragment})/o
+  TagAttributes               = /(#{WordRegex}+)\s*\:\s*(#{QuotedFragment})/o
   AnyStartingTag              = /\{\{|\{\%/
   PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/o
   TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/o

lib/liquid/block.rb

@@ -3,7 +3,7 @@ module Liquid
   class Block < Tag
     IsTag             = /^#{TagStart}/o
     IsVariable        = /^#{VariableStart}/o
-    FullToken         = /^#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}$/o
+    FullToken         = /^#{TagStart}\s*(#{WordRegex}+)\s*(.*)?#{TagEnd}$/o
     ContentOfVariable = /^#{VariableStart}(.*)#{VariableEnd}$/o
 
     def parse(tokens)

lib/liquid/context.rb

@@ -1,5 +1,6 @@
 module Liquid
 
+  
   # Context keeps the variable stack and resolves variables, as well as keywords
   #
   #   context['variable'] = 'testing'
@@ -24,6 +25,7 @@ module Liquid
       squash_instance_assigns_with_environments
 
       @interrupts = []
+      @variable_cache = {}
     end
 
     def strainer
@@ -77,17 +79,20 @@ module Liquid
 
     # Push new local scope on the stack. use <tt>Context#stack</tt> instead
     def push(new_scope={})
+      @variable_cache = {}
       @scopes.unshift(new_scope)
       raise StackLevelError, "Nesting too deep" if @scopes.length > 100
     end
 
     # Merge a hash of variables in the current local scope
     def merge(new_scopes)
+      @variable_cache = {}
       @scopes[0].merge!(new_scopes)
     end
 
     # Pop from the stack. use <tt>Context#stack</tt> instead
     def pop
+      @variable_cache = {}
       raise ContextError if @scopes.size == 1
       @scopes.shift
     end
@@ -101,69 +106,112 @@ module Liquid
     #
     #   context['var]  #=> nil
     def stack(new_scope={})
+      @variable_cache = {}
       push(new_scope)
       yield
     ensure
       pop
+      @variable_cache = {}
     end
 
     def clear_instance_assigns
+      @variable_cache = {}
       @scopes[0] = {}
     end
 
-    # Only allow String, Numeric, Hash, Array, Proc, Boolean or <tt>Liquid::Drop</tt>
-    def []=(key, value)
-      @scopes[0][key] = value
+
+    # Look up variable, either resolve directly after considering the name. We can directly handle
+    # Strings, digits, floats and booleans (true,false).
+    # If no match is made we lookup the variable in the current scope and
+    # later move up to the parent blocks to see if we can resolve the variable somewhere up the tree.
+    # Some special keywords return symbols. Those symbols are to be called on the rhs object in expressions
+    #
+    # Example:
+    #   products == empty #=> products.empty?
+    def resolve(key)        
+      case key
+      when nil, ""
+        return nil
+      when "blank"
+        return :blank?
+      when "empty"
+        return :empty?
+      end
+      
+      result = Parser.parse(key)   
+      stack = []
+
+      result.each do |(sym, value)|          
+
+        case sym
+        when :id
+          stack.push value
+        when :lookup
+          left = stack.pop
+          value = find_variable(left)
+          
+          stack.push(harden(value))
+        when :range
+          right = stack.pop.to_i
+          left  = stack.pop.to_i
+          
+          stack.push (left..right)
+        when :buildin
+          left = stack.pop
+          value = invoke_buildin(left, value)
+          
+          stack.push(harden(value))
+        when :call
+          left = stack.pop
+          right = stack.pop
+          value = lookup_and_evaluate(right, left)
+
+          stack.push(harden(value))
+        else 
+          raise "unknown #{sym}"
+        end
+      end
+
+      return stack.first
     end
 
-    def [](key)
-      resolve(key)
+
+    # Only allow String, Numeric, Hash, Array, Proc, Boolean or <tt>Liquid::Drop</tt>
+    def []=(key, value)
+      @variable_cache[key] = value
+      @scopes[0][key] = value
     end
 
     def has_key?(key)
       resolve(key) != nil
     end
 
-    private
-      LITERALS = {
-        nil => nil, 'nil' => nil, 'null' => nil, '' => nil,
-        'true'  => true,
-        'false' => false,
-        'blank' => :blank?,
-        'empty' => :empty?
-      }
+    alias_method :[], :resolve
 
-      # Look up variable, either resolve directly after considering the name. We can directly handle
-      # Strings, digits, floats and booleans (true,false).
-      # If no match is made we lookup the variable in the current scope and
-      # later move up to the parent blocks to see if we can resolve the variable somewhere up the tree.
-      # Some special keywords return symbols. Those symbols are to be called on the rhs object in expressions
-      #
-      # Example:
-      #   products == empty #=> products.empty?
-      def resolve(key)
-        if LITERALS.key?(key)
-          LITERALS[key]
+    private
+
+      def invoke_buildin(obj, key)
+        # as weird as this is, liquid unit tests demand that we prioritize hash lookups 
+        # to buildins. So if we got a hash and it has a :first element we need to call that 
+        # instead of sending the first message...
+
+        if obj.respond_to?(:has_key?) && obj.has_key?(key)
+          return lookup_and_evaluate(obj, key)
+        end
+
+        if obj.respond_to?(key) 
+          return obj.send(key)
         else
-          case key
-          when /^'(.*)'$/ # Single quoted strings
-            $1
-          when /^"(.*)"$/ # Double quoted strings
-            $1
-          when /^(-?\d+)$/ # Integer and floats
-            $1.to_i
-          when /^\((\S+)\.\.(\S+)\)$/ # Ranges
-            (resolve($1).to_i..resolve($2).to_i)
-          when /^(-?\d[\d\.]+)$/ # Floats
-            $1.to_f
-          else
-            variable(key)
-          end
+          return nil    
         end
       end
 
       # Fetches an object starting at the local scope and then moving up the hierachy
       def find_variable(key)
+        if val = @variable_cache[key]
+          return val
+        end
+
         scope = @scopes.find { |s| s.has_key?(key) }
 
         if scope.nil?
@@ -177,72 +225,36 @@ module Liquid
 
         scope     ||= @environments.last || @scopes.last
         variable  ||= lookup_and_evaluate(scope, key)
-
-        variable = variable.to_liquid
-        variable.context = self if variable.respond_to?(:context=)
+        @variable_cache[key] = variable
 
         return variable
       end
 
-      # Resolves namespaced queries gracefully.
-      #
-      # Example
-      #  @context['hash'] = {"name" => 'tobi'}
-      #  assert_equal 'tobi', @context['hash.name']
-      #  assert_equal 'tobi', @context['hash["name"]']
-      def variable(markup)
-        parts = markup.scan(VariableParser)
-        square_bracketed = /^\[(.*)\]$/
-
-        first_part = parts.shift
-
-        if first_part =~ square_bracketed
-          first_part = resolve($1)
-        end
-
-        if object = find_variable(first_part)
-
-          parts.each do |part|
-            part = resolve($1) if part_resolved = (part =~ square_bracketed)
-
-            # If object is a hash- or array-like object we look for the
-            # presence of the key and if its available we return it
-            if object.respond_to?(:[]) and
-              ((object.respond_to?(:has_key?) and object.has_key?(part)) or
-               (object.respond_to?(:fetch) and part.is_a?(Integer)))
-
-              # if its a proc we will replace the entry with the proc
-              res = lookup_and_evaluate(object, part)
-              object = res.to_liquid
-
-              # Some special cases. If the part wasn't in square brackets and
-              # no key with the same name was found we interpret following calls
-              # as commands and call them on the current object
-            elsif !part_resolved and object.respond_to?(part) and ['size', 'first', 'last'].include?(part)
-
-              object = object.send(part.intern).to_liquid
-
-              # No key was present with the desired value and it wasn't one of the directly supported
-              # keywords either. The only thing we got left is to return nil
-            else
-              return nil
-            end
-
-            # If we are dealing with a drop here we have to
-            object.context = self if object.respond_to?(:context=)
-          end
-        end
-
-        object
-      end # variable
-
       def lookup_and_evaluate(obj, key)
-        if (value = obj[key]).is_a?(Proc) && obj.respond_to?(:[]=)
-          obj[key] = (value.arity == 0) ? value.call : value.call(self)
-        else
-          value
+        return nil unless obj.respond_to?(:[])
+        
+        if obj.is_a?(Array)
+          return nil unless key.is_a?(Integer)
         end
-      end # lookup_and_evaluate
+
+        value = obj[key]
+
+        if value.is_a?(Proc)      
+          # call the proc
+          value = (value.arity == 0) ? value.call : value.call(self)
+
+          # memozie if possible
+          obj[key] = value if obj.respond_to?(:[]=)
+        end
+
+        value
+      end
+
+      def harden(value)
+        value = value.to_liquid
+        value.context = self if value.respond_to?(:context=)
+        return value
+      end
 
       def squash_instance_assigns_with_environments
         @scopes.last.each_key do |k|
@@ -254,6 +266,7 @@ module Liquid
           end
         end
       end # squash_instance_assigns_with_environments
+
   end # Context
 
 end # Liquid

lib/liquid/drop.rb

@@ -54,7 +54,7 @@ module Liquid
 
     # Check for method existence without invoking respond_to?, which creates symbols
     def self.invokable?(method_name)
-      @invokable_methods ||= Set.new(["to_liquid"] + (public_instance_methods - Liquid::Drop.public_instance_methods).map(&:to_s))
+      @invokable_methods ||= Set.new((public_instance_methods - Liquid::Drop.public_instance_methods).map(&:to_s))
       @invokable_methods.include?(method_name.to_s)
     end
   end

lib/liquid/htmltags.rb

@@ -1,6 +1,6 @@
 module Liquid
   class TableRow < Block
-    Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)/o
+    Syntax = /(#{WordRegex}+)\s+in\s+(#{QuotedFragment}+)/o
 
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax
@@ -43,6 +43,7 @@ module Liquid
             'index0'  => index,
             'col'     => col + 1,
             'col0'    => col,
+            'index0'  => index,
             'rindex'  => length - index,
             'rindex0' => length - index - 1,
             'first'   => (index == 0),

lib/liquid/standardfilters.rb

@@ -93,26 +93,21 @@ module Liquid
     # map/collect on a given property
     def map(input, property)
       ary = [input].flatten
-      ary.map do |e|
-        e = e.call if e.is_a?(Proc)
-        e = e.to_liquid if e.respond_to?(:to_liquid)
-
-        if property == "to_liquid"
-          e
-        elsif e.respond_to?(:[])
-          e[property]
-        end
+      if ary.first.respond_to?('[]') and !ary.first[property].nil?
+        ary.map {|e| e[property] }
+      elsif ary.first.respond_to?(property)
+        ary.map {|e| e.send(property) }
       end
     end
 
     # Replace occurrences of a string with another
     def replace(input, string, replacement = '')
-      input.to_s.gsub(string, replacement.to_s)
+      input.to_s.gsub(string, replacement)
     end
 
     # Replace the first occurrences of a string with another
     def replace_first(input, string, replacement = '')
-      input.to_s.sub(string, replacement.to_s)
+      input.to_s.sub(string, replacement)
     end
 
     # remove a substring

lib/liquid/tags/capture.rb

@@ -12,7 +12,7 @@ module Liquid
   # in a sidebar or footer.
   #
   class Capture < Block
-    Syntax = /(\w+)/
+    Syntax = /(#{WordRegex}+)/o
 
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax

lib/liquid/tags/for.rb

@@ -44,7 +44,7 @@ module Liquid
   # forloop.last:: Returns true if the item is the last item.
   #
   class For < Block                                             
-    Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o
+    Syntax = /(#{WordRegex}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/ou
   
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax

lib/liquid/tags/if.rb

@@ -15,7 +15,6 @@ module Liquid
     SyntaxHelp = "Syntax Error in tag 'if' - Valid syntax: if [expression]"
     Syntax = /(#{QuotedFragment})\s*([=!<>a-z_]+)?\s*(#{QuotedFragment})?/o
     ExpressionsAndOperators = /(?:\b(?:\s?and\s?|\s?or\s?)\b|(?:\s*(?!\b(?:\s?and\s?|\s?or\s?)\b)(?:#{QuotedFragment}|\S+)\s*)+)/o
-    BOOLEAN_OPERATORS = %w(and or)
 
     def initialize(tag_name, markup, tokens)
       @blocks = []
@@ -62,8 +61,7 @@ module Liquid
             raise(SyntaxError, SyntaxHelp) unless expressions.shift.to_s =~ Syntax
 
             new_condition = Condition.new($1, $2, $3)
-            raise SyntaxError, "invalid boolean operator" unless BOOLEAN_OPERATORS.include?(operator)
-            new_condition.send(operator, condition)
+            new_condition.send(operator.to_sym, condition)
             condition = new_condition
           end
 
@@ -73,6 +71,8 @@ module Liquid
         @blocks.push(block)
         @nodelist = block.attach(Array.new)
       end
+
+
   end
 
   Template.register_tag('if', If)

lib/liquid/tags/include.rb

@@ -2,10 +2,10 @@ module Liquid
   class Include < Tag
     Syntax = /(#{QuotedFragment}+)(\s+(?:with|for)\s+(#{QuotedFragment}+))?/o
   
-    def initialize(tag_name, markup, tokens)      
+    def initialize(tag_name, markup, tokens)
       if markup =~ Syntax
 
-        @template_name = $1        
+        @template_name = $1
         @variable_name = $3
         @attributes    = {}
 
@@ -24,8 +24,7 @@ module Liquid
     end
     
     def render(context)
-      source = _read_template_from_file_system(context)
-      partial = Liquid::Template.parse(source)
+      partial = load_cached_partial(context)
       variable = context[@variable_name || @template_name[1..-2]]
       
       context.stack do
@@ -46,7 +45,21 @@ module Liquid
     end
    
     private
-      def _read_template_from_file_system(context)
+      def load_cached_partial(context)
+        cached_partials = context.registers[:cached_partials] || {}
+        template_name = context[@template_name]
+
+        if cached = cached_partials[template_name]
+          return cached
+        end
+        source = read_template_from_file_system(context)
+        partial = Liquid::Template.parse(source)
+        cached_partials[template_name] = partial
+        context.registers[:cached_partials] = cached_partials
+        partial
+      end
+
+      def read_template_from_file_system(context)
         file_system = context.registers[:file_system] || Liquid::Template.file_system
       
         # make read_template_file call backwards-compatible.

lib/liquid/variable.rb

@@ -23,7 +23,7 @@ module Liquid
         if match[2].match(/#{FilterSeparator}\s*(.*)/o)
           filters = Regexp.last_match(1).scan(FilterParser)
           filters.each do |f|
-            if matches = f.match(/\s*(\w+)(?:\s*#{FilterArgumentSeparator}(.*))?/)
+            if matches = f.match(/\s*(#{WordRegex}+)(?:\s*#{FilterArgumentSeparator}(.*))?/)
               filtername = matches[1]
               filterargs = matches[2].to_s.scan(/(?:\A|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o).flatten
               @filters << [filtername, filterargs]

liquid.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = "liquid"
-  s.version     = "2.5.5"
+  s.version     = "2.5.0"
   s.platform    = Gem::Platform::RUBY
   s.summary     = "A secure, non-evaling end user template engine with aesthetic markup."
   s.authors     = ["Tobias Luetke"]

performance/theme_runner.rb

@@ -14,6 +14,17 @@ require File.dirname(__FILE__) + '/shopify/liquid'
 require File.dirname(__FILE__) + '/shopify/database.rb'
 
 class ThemeRunner
+  class FileSystem
+
+    def initialize(path)
+      @path = path
+    end
+
+    # Called by Liquid to retrieve a template file
+    def read_template_file(template_path, context)
+      File.read(@path + '/' + template_path + '.liquid')
+    end
+  end
 
   # Load all templates into memory, do this now so that 
   # we don't profile IO. 
@@ -47,7 +58,7 @@ class ThemeRunner
 
       # Compute page_tempalte outside of profiler run, uninteresting to profiler
       page_template = File.basename(template_name, File.extname(template_name))
-      compile_and_render(liquid, layout, assigns, page_template) 
+      compile_and_render(liquid, layout, assigns, page_template, template_name)
        
     end
   end
@@ -74,7 +85,7 @@ class ThemeRunner
       html = nil
 
       RubyProf.resume
-      html = compile_and_render(liquid, layout, assigns, page_template) 
+      html = compile_and_render(liquid, layout, assigns, page_template, template_name)
       RubyProf.pause
       
 
@@ -88,10 +99,11 @@ class ThemeRunner
     RubyProf.stop
   end
 
-  def compile_and_render(template, layout, assigns, page_template)    
+  def compile_and_render(template, layout, assigns, page_template, template_file)
     tmpl = Liquid::Template.new
     tmpl.assigns['page_title'] = 'Page title'
     tmpl.assigns['template'] = page_template
+    tmpl.registers[:file_system] = ThemeRunner::FileSystem.new(File.dirname(template_file))
 
     content_for_layout = tmpl.parse(template).render(assigns)
 

test/liquid/assign_test.rb

@@ -12,7 +12,13 @@ class AssignTest < Test::Unit::TestCase
                            '{% assign foo = values %}.{{ foo[1] }}.',
                            'values' => %w{foo bar baz})
   end
-  
+
+  def test_assigned_utf8_variable
+    assert_template_result('.bar.',
+                           "{% assign foo\u6000 = values %}.{{ foo\u6000[1] }}.",
+                           'values' => %w{foo bar baz})
+  end
+
   def test_assign_with_filter
     assert_template_result('.bar.',
                            '{% assign foo = values | split: "," %}.{{ foo[1] }}.',

test/liquid/block_test.rb

@@ -51,6 +51,14 @@ class BlockTest < Test::Unit::TestCase
     end
   end
 
+  def test_with_custom_utf8_tag
+    Liquid::Template.register_tag("testtag\u6000", Block)
+
+    assert_nothing_thrown do
+      template = Liquid::Template.parse( "{% testtag\u6000 something\u6000 %} {% endtesttag\u6000 %}")
+    end
+  end
+
   private
     def block_types(nodelist)
       nodelist.collect { |node| node.class }

test/liquid/capture_test.rb

@@ -7,6 +7,10 @@ class CaptureTest < Test::Unit::TestCase
     assert_template_result("test string", "{% capture 'var' %}test string{% endcapture %}{{var}}", {})
   end
 
+  def test_captures_block_content_in_utf8_variable
+    assert_template_result("test string", "{% capture var\u6000 %}test string{% endcapture %}{{var\u6000}}", {})
+  end
+
   def test_capture_to_variable_from_outer_scope_if_existing
     template_source = <<-END_TEMPLATE
     {% assign var = '' %}

test/liquid/context_test.rb

@@ -98,6 +98,11 @@ class ContextTest < Test::Unit::TestCase
     assert_equal nil, @context['nil']
   end
 
+  def test_utf8_variables
+    @context["chinese\u6000variable"] = 'chinese'
+    assert_equal 'chinese', @context["chinese\u6000variable"]
+  end
+
   def test_variables_not_existing
     assert_equal nil, @context['does_not_exist']
   end

test/liquid/drop_test.rb

@@ -71,34 +71,23 @@ class DropsTest < Test::Unit::TestCase
   include Liquid
 
   def test_product_drop
+
     assert_nothing_raised do
       tpl = Liquid::Template.parse( '  '  )
       tpl.render('product' => ProductDrop.new)
     end
   end
 
-  def test_drop_does_only_respond_to_whitelisted_methods
-    assert_equal "", Liquid::Template.parse("{{ product.inspect }}").render('product' => ProductDrop.new)
-    assert_equal "", Liquid::Template.parse("{{ product.pretty_inspect }}").render('product' => ProductDrop.new)
-    assert_equal "", Liquid::Template.parse("{{ product.whatever }}").render('product' => ProductDrop.new)
-    assert_equal "", Liquid::Template.parse('{{ product | map: "inspect" }}').render('product' => ProductDrop.new)
-    assert_equal "", Liquid::Template.parse('{{ product | map: "pretty_inspect" }}').render('product' => ProductDrop.new)
-    assert_equal "", Liquid::Template.parse('{{ product | map: "whatever" }}').render('product' => ProductDrop.new)
-  end
-
-  def test_drops_respond_to_to_liquid
-    assert_equal "text1", Liquid::Template.parse("{{ product.to_liquid.texts.text }}").render('product' => ProductDrop.new)
-    assert_equal "text1", Liquid::Template.parse('{{ product | map: "to_liquid" | map: "texts" | map: "text" }}').render('product' => ProductDrop.new)
-  end
-
   def test_text_drop
     output = Liquid::Template.parse( ' {{ product.texts.text }} '  ).render('product' => ProductDrop.new)
     assert_equal ' text1 ', output
+
   end
 
   def test_unknown_method
     output = Liquid::Template.parse( ' {{ product.catchall.unknown }} '  ).render('product' => ProductDrop.new)
     assert_equal ' method: unknown ', output
+
   end
 
   def test_integer_argument_drop

test/liquid/standard_filter_test.rb

@@ -4,27 +4,6 @@ class Filters
   include Liquid::StandardFilters
 end
 
-class TestThing
-  def initialize
-    @foo = 0
-  end
-
-  def to_s
-    "woot: #{@foo}"
-  end
-
-  def to_liquid
-    @foo += 1
-    self
-  end
-end
-
-class TestDrop < Liquid::Drop
-  def test
-    "testfoo"
-  end
-end
-
 class StandardFiltersTest < Test::Unit::TestCase
   include Liquid
 
@@ -107,23 +86,6 @@ class StandardFiltersTest < Test::Unit::TestCase
       'ary' => [{'foo' => {'bar' => 'a'}}, {'foo' => {'bar' => 'b'}}, {'foo' => {'bar' => 'c'}}]
   end
 
-  def test_map_doesnt_call_arbitrary_stuff
-    assert_equal "", Liquid::Template.parse('{{ "foo" | map: "__id__" }}').render
-    assert_equal "", Liquid::Template.parse('{{ "foo" | map: "inspect" }}').render
-  end
-
-  def test_map_calls_to_liquid
-    t = TestThing.new
-    assert_equal "woot: 1", Liquid::Template.parse('{{ foo }}').render("foo" => t)
-  end
-
-  def test_map_over_proc
-    drop = TestDrop.new
-    p = Proc.new{ drop }
-    templ = '{{ procs | map: "test" }}'
-    assert_equal "testfoo", Liquid::Template.parse(templ).render("procs" => [p])
-  end
-
   def test_date
     assert_equal 'May', @filters.date(Time.parse("2006-05-05 10:00:00"), "%B")
     assert_equal 'June', @filters.date(Time.parse("2006-06-05 10:00:00"), "%B")
@@ -157,9 +119,9 @@ class StandardFiltersTest < Test::Unit::TestCase
   end
 
   def test_replace
-    assert_equal '2 2 2 2', @filters.replace('1 1 1 1', '1', 2)
-    assert_equal '2 1 1 1', @filters.replace_first('1 1 1 1', '1', 2)
-    assert_template_result '2 1 1 1', "{{ '1 1 1 1' | replace_first: '1', 2 }}"
+    assert_equal 'b b b b', @filters.replace("a a a a", 'a', 'b')
+    assert_equal 'b a a a', @filters.replace_first("a a a a", 'a', 'b')
+    assert_template_result 'b a a a', "{{ 'a a a a' | replace_first: 'a', 'b' }}"
   end
 
   def test_remove

test/liquid/tags/for_tag_test.rb

@@ -25,6 +25,11 @@ HERE
     assert_template_result(expected,template,'array' => [1,2,3])
   end
 
+  def test_utf8_for
+    assigns = {"array\u6000chinese" => [1,2,3]}
+    assert_template_result('123', "{% for item\u6000chinese in array\u6000chinese %}{{ item\u6000chinese }}{% endfor %}", assigns)
+  end
+
   def test_for_reversed
     assigns = {'array' => [ 1, 2, 3] }
     assert_template_result('321','{%for item in array reversed %}{{item}}{%endfor%}',assigns)

test/liquid/tags/html_tag_test.rb

@@ -26,6 +26,12 @@ class HtmlTagTest < Test::Unit::TestCase
                            'numbers' => [])
   end
 
+  def test_utf8_html_table
+    assert_template_result("<tr class=\"row1\">\n<td class=\"col1\"> 1 </td></tr>\n",
+                           "{% tablerow n\u6000 in numbers\u6000 %} {{n\u6000}} {% endtablerow %}",
+                           "numbers\u6000" => [1])
+  end
+
   def test_html_table_with_different_cols
     assert_template_result("<tr class=\"row1\">\n<td class=\"col1\"> 1 </td><td class=\"col2\"> 2 </td><td class=\"col3\"> 3 </td><td class=\"col4\"> 4 </td><td class=\"col5\"> 5 </td></tr>\n<tr class=\"row2\"><td class=\"col1\"> 6 </td></tr>\n",
                            '{% tablerow n in numbers cols:5%} {{n}} {% endtablerow %}',

test/liquid/tags/if_else_tag_test.rb

@@ -157,10 +157,4 @@ class IfElseTagTest < Test::Unit::TestCase
     assert_template_result('yes',
                            %({% if 'gnomeslab-and-or-liquid' contains 'gnomeslab-and-or-liquid' %}yes{% endif %}))
   end
-
-  def test_operators_are_whitelisted
-    assert_raise(SyntaxError) do
-      assert_template_result('', %({% if 1 or throw or or 1 %}yes{% endif %}))
-    end
-  end
 end # IfElseTest

test/liquid/tags/include_tag_test.rb

@@ -39,6 +39,15 @@ class OtherFileSystem
   end
 end
 
+class CountingFileSystem
+  attr_reader :count
+  def read_template_file(template_path, context)
+    @count ||= 0
+    @count += 1
+    'from CountingFileSystem'
+  end
+end
+
 class IncludeTagTest < Test::Unit::TestCase
   include Liquid
 
@@ -136,4 +145,22 @@ class IncludeTagTest < Test::Unit::TestCase
 
     assert_equal "Product: Draft 151cm ", Template.parse("{% include template for product %}").render("template" => 'product', 'product' => { 'title' => 'Draft 151cm'})
   end
-end # IncludeTagTest
+
+  def test_include_tag_caches_second_read_of_same_partial
+    file_system = CountingFileSystem.new
+    assert_equal 'from CountingFileSystemfrom CountingFileSystem',
+      Template.parse("{% include 'pick_a_source' %}{% include 'pick_a_source' %}").render({}, :registers => {:file_system => file_system})
+    assert_equal 1, file_system.count
+  end
+
+  def test_include_tag_doesnt_cache_partials_across_renders
+    file_system = CountingFileSystem.new
+    assert_equal 'from CountingFileSystem',
+      Template.parse("{% include 'pick_a_source' %}").render({}, :registers => {:file_system => file_system})
+    assert_equal 1, file_system.count
+
+    assert_equal 'from CountingFileSystem',
+      Template.parse("{% include 'pick_a_source' %}").render({}, :registers => {:file_system => file_system})
+    assert_equal 2, file_system.count
+  end
+end # IncludeTagTest

test/liquid/variable_test.rb

@@ -50,6 +50,12 @@ class VariableTest < Test::Unit::TestCase
     assert_equal [["things",["\"%Y, okay?\"","'the other one'"]]], var.filters
   end
 
+  def test_utf8_filters
+    var = Variable.new("foo | chinese\u6000filter: value\u6000")
+    assert_equal 'foo', var.name
+    assert_equal [["chinese\u6000filter",["value\u6000"]]], var.filters
+  end
+
   def test_filter_with_date_parameter
 
     var = Variable.new(%! '2006-06-06' | date: "%m/%d/%Y"!)