Implement naive recusrive descent

Ragel doesn't allow us to recurse so we simply
reinvoke the parser for each step.

Rakefile

@@ -7,6 +7,11 @@ require 'rubygems/package_task'
 
 task :default => 'test'
 
+task :ragel do
+  sh "find . -name '*.rl' | xargs ragel -R -F1"
+end
+
+
 Rake::TestTask.new(:test) do |t|
   t.libs << '.' << 'lib' << 'test'
   t.test_files = FileList['test/liquid/**/*_test.rb']

lib/liquid.rb

@@ -62,6 +62,7 @@ require 'liquid/standardfilters'
 require 'liquid/condition'
 require 'liquid/module_ex'
 require 'liquid/utils'
+require 'liquid/parser'
 
 # Load all the tags of the standard library
 #

lib/liquid/context.rb

@@ -1,5 +1,6 @@
 module Liquid
 
+  
   # Context keeps the variable stack and resolves variables, as well as keywords
   #
   #   context['variable'] = 'testing'
@@ -128,13 +129,6 @@ module Liquid
     end
 
     private
-      LITERALS = {
-        nil => nil, 'nil' => nil, 'null' => nil, '' => nil,
-        'true'  => true,
-        'false' => false,
-        'blank' => :blank?,
-        'empty' => :empty?
-      }
 
       # Look up variable, either resolve directly after considering the name. We can directly handle
       # Strings, digits, floats and booleans (true,false).
@@ -144,25 +138,67 @@ module Liquid
       #
       # Example:
       #   products == empty #=> products.empty?
-      def resolve(key)
-        if LITERALS.key?(key)
-          LITERALS[key]
-        else
-          case key
-          when /^'(.*)'$/ # Single quoted strings
-            $1
-          when /^"(.*)"$/ # Double quoted strings
-            $1
-          when /^(-?\d+)$/ # Integer and floats
-            $1.to_i
-          when /^\((\S+)\.\.(\S+)\)$/ # Ranges
-            (resolve($1).to_i..resolve($2).to_i)
-          when /^(-?\d[\d\.]+)$/ # Floats
-            $1.to_f
-          else
-            variable(key)
+      def resolve(key)        
+        case key
+        when nil, ""
+          return nil
+        when "blank"
+          return :blank?
+        when "empty"
+          return :empty?
+        end
+        
+        result = Parser.parse(key)   
+        stack = []
+
+        result.each do |(sym, value)|          
+
+          case sym
+          when :id
+            stack.push value
+          when :lookup
+            left = stack.pop
+            value = find_variable(left)
+            
+            stack.push(harden(value))
+          when :range
+            right = stack.pop.to_i
+            left  = stack.pop.to_i
+            
+            stack.push (left..right)
+          when :buildin
+            left = stack.pop
+            value = invoke_buildin(left, value)
+            
+            stack.push(harden(value))
+          when :call
+            left = stack.pop
+            right = stack.pop
+            value = lookup_and_evaluate(right, left)
+
+            stack.push(harden(value))
+          else 
+            raise "unknown #{sym}"
           end
         end
+
+        return stack.first
+      end
+
+      def invoke_buildin(obj, key)
+        # as weird as this is, liquid unit tests demand that we prioritize hash lookups 
+        # to buildins. So if we got a hash and it has a :first element we need to call that 
+        # instead of sending the first message...
+
+        if obj.respond_to?(:has_key?) && obj.has_key?(key)
+          return lookup_and_evaluate(obj, key)
+        end
+
+        if obj.respond_to?(key) 
+          return obj.send(key)
+        else
+          return nil    
+        end
       end
 
       # Fetches an object starting at the local scope and then moving up the hierachy
@@ -181,71 +217,35 @@ module Liquid
         scope     ||= @environments.last || @scopes.last
         variable  ||= lookup_and_evaluate(scope, key)
 
-        variable = variable.to_liquid
-        variable.context = self if variable.respond_to?(:context=)
-
         return variable
       end
 
-      # Resolves namespaced queries gracefully.
-      #
-      # Example
-      #  @context['hash'] = {"name" => 'tobi'}
-      #  assert_equal 'tobi', @context['hash.name']
-      #  assert_equal 'tobi', @context['hash["name"]']
-      def variable(markup)
-        parts = markup.scan(VariableParser)
-        square_bracketed = /^\[(.*)\]$/
-
-        first_part = parts.shift
-
-        if first_part =~ square_bracketed
-          first_part = resolve($1)
-        end
-
-        if object = find_variable(first_part)
-
-          parts.each do |part|
-            part = resolve($1) if part_resolved = (part =~ square_bracketed)
-
-            # If object is a hash- or array-like object we look for the
-            # presence of the key and if its available we return it
-            if object.respond_to?(:[]) and
-              ((object.respond_to?(:has_key?) and object.has_key?(part)) or
-               (object.respond_to?(:fetch) and part.is_a?(Integer)))
-
-              # if its a proc we will replace the entry with the proc
-              res = lookup_and_evaluate(object, part)
-              object = res.to_liquid
-
-              # Some special cases. If the part wasn't in square brackets and
-              # no key with the same name was found we interpret following calls
-              # as commands and call them on the current object
-            elsif !part_resolved and object.respond_to?(part) and ['size', 'first', 'last'].include?(part)
-
-              object = object.send(part.intern).to_liquid
-
-              # No key was present with the desired value and it wasn't one of the directly supported
-              # keywords either. The only thing we got left is to return nil
-            else
-              return nil
-            end
-
-            # If we are dealing with a drop here we have to
-            object.context = self if object.respond_to?(:context=)
-          end
-        end
-
-        object
-      end # variable
-
       def lookup_and_evaluate(obj, key)
-        if (value = obj[key]).is_a?(Proc) && obj.respond_to?(:[]=)
-          obj[key] = (value.arity == 0) ? value.call : value.call(self)
-        else
-          value
+        return nil unless obj.respond_to?(:[])
+        
+        if obj.is_a?(Array)
+          return nil unless key.is_a?(Integer)
         end
-      end # lookup_and_evaluate
+
+        value = obj[key]
+
+        case value
+        when Proc
+          # call the proc
+          value = (value.arity == 0) ? value.call : value.call(self)
+
+          # memozie if possible
+          obj[key] = value if obj.respond_to?(:[]=)
+        end
+
+        value
+      end
+
+      def harden(value)
+        value = value.to_liquid
+        value.context = self if value.respond_to?(:context=)
+        return value
+      end
 
       def squash_instance_assigns_with_environments
         @scopes.last.each_key do |k|
@@ -257,6 +257,7 @@ module Liquid
           end
         end
       end # squash_instance_assigns_with_environments
+
   end # Context
 
 end # Liquid

lib/liquid/parser.rl

@@ -0,0 +1,126 @@
+# Parser for context#[] method. Generated through ragel from parser.rl 
+# Only modify parser.rl. Run rake ragel afterwards to generate this file. 
+#
+#VERBOSE=true
+
+%%{
+  machine fsm;
+
+  action mark {
+    mark = p
+  }
+
+  action lookup {
+    emit(:lookup, :instruction, nil, tokens)
+  }
+
+  action call {
+    emit(:call, :instruction, nil, tokens)
+  }
+  action range {
+    emit(:range, :instruction, nil, tokens)
+  }
+
+  constants = ( "true" | "false" | "nil" | "null" );
+
+  # strings 
+  string = "\"" any* "\"" | "'" any* "'";
+
+  # nothingness
+  nil = "nil" | "null" ;
+
+  # numbers
+  integer = ('+'|'-')? digit+;
+  float = ('+'|'-')? digit+ '.' digit+;
+
+  # simple values
+  primitive = (
+
+    integer       >mark %{ emit(:id, :integer, Integer(data[mark..p-1]), tokens) } |
+
+    float         >mark %{ emit(:id, :float, Float(data[mark..p-1]), tokens) } |
+
+    nil           %{ emit(:id, :nil, nil, tokens) } | 
+    "true"        %{ emit(:id, :bool, true, tokens) } |
+    "false"       %{ emit(:id, :bool, false, tokens)} |
+    
+    string        >mark %{ emit(:id, :string, data[mark+1..p-2], tokens) } 
+
+  );
+
+  entity = (
+    ((alpha [A-Za-z0-9_\-]*) - (constants)) >mark %{ 
+      emit(:id, :label, data[mark..p-1], tokens) 
+      emit(:lookup, :variable, nil, tokens) 
+    }
+  );
+
+  # Because of recursion we cannot immediatly resolve the content of this in 
+  # the current grammar. We simply re-invoke the parser here to descend into 
+  # the substring
+  recur = (
+    (any+ - ']') >mark %{       
+      self.parse(data[mark..p-1], tokens)
+    }
+  );
+
+  expr = (
+    entity |
+    primitive |    
+    "(" (primitive | entity) ".." (primitive | entity) <: ")"  %range |
+    "[" recur "]" %lookup    
+  );
+
+  hash_accessors  = (
+    "[" recur "]" %call |
+
+    ".first"  %{ 
+      emit(:buildin, :symbol, "first", tokens)       
+    } |
+
+    ".last"  %{ 
+      emit(:buildin, :symbol, "last", tokens)       
+    } |
+
+    ".size"  %{ 
+      emit(:buildin, :symbol, "size", tokens)       
+    } |
+
+    "." ((alpha [A-Za-z0-9_\-]*) - ("first"|"last"|"size"))  >mark  %{ 
+      emit(:id, :label, data[mark..p-1], tokens) 
+      emit(:call, :variable, nil, tokens) 
+    }
+  );
+
+  main := ( 
+    
+    expr <: (hash_accessors)*
+
+  );
+    
+}%%
+# % fix syntax highlighting
+
+
+module Liquid
+  module Parser
+    %% write data;
+
+    def self.emit(sym, type, data, tokens) 
+      puts "emitting: #{type} #{sym} -> #{data.inspect}" if $VERBOSE
+      tokens.push [sym, data]
+    end
+
+    def self.parse(data, tokens = [])      
+      puts "--> self.parse with #{data.inspect}, #{tokens.inspect}" if $VERBOSE
+
+      eof = data.length  
+
+      %% write init;
+      %% write exec;   
+
+      puts "<-- #{tokens.inspect}" if $VERBOSE
+      return tokens 
+    end
+  end
+end

performance/benchmark.rb

@@ -5,6 +5,7 @@ require File.dirname(__FILE__) + '/theme_runner'
 profiler = ThemeRunner.new
 
 Benchmark.bmbm do |x|
-  x.report("parse & run:")   { 10.times { profiler.run(false) } }
+  x.report("parse:")   { 100.times { profiler.compile } }
+  x.report("parse & run:")   { 100.times { profiler.run } }
 end
 

performance/profile.rb

@@ -6,14 +6,12 @@ profiler = ThemeRunner.new
 
 puts 'Running profiler...'
 
-results  = profiler.run
+results  = profiler.run_profile
 
 puts 'Success'
-puts
 
-[RubyProf::FlatPrinter, RubyProf::GraphPrinter, RubyProf::GraphHtmlPrinter, RubyProf::CallTreePrinter].each do |klass|
-  filename = (ENV['TMP'] || '/tmp') + (klass.name.include?('Html') ? "/liquid.#{klass.name.downcase}.html" : "/callgrind.liquid.#{klass.name.downcase}.txt")
-  filename.gsub!(/:+/, '_')
-  File.open(filename, "w+") { |fp| klass.new(results).print(fp, :print_file => true) }
-  $stderr.puts "wrote #{klass.name} output to #{filename}"
+filename = (ENV['TMP'] || '/tmp') + "/callgrind.liquid.txt"
+File.open(filename, "w+") do |fp| 
+  RubyProf::CallTreePrinter.new(results).print(fp, :print_file => true) 
 end
+$stderr.puts "wrote RubyProf::CallTreePrinter output to #{filename}"

performance/theme_runner.rb

@@ -27,8 +27,33 @@ class ThemeRunner
     end.compact
   end  
 
+  def compile
+    # Dup assigns because will make some changes to them
 
-  def run()
+    @tests.each do |liquid, layout, template_name|
+
+      tmpl = Liquid::Template.new
+      tmpl.parse(liquid)
+      tmpl = Liquid::Template.new
+      tmpl.parse(layout)
+    end
+  end
+
+   def run
+    # Dup assigns because will make some changes to them
+    assigns = Database.tables.dup
+
+    @tests.each do |liquid, layout, template_name|
+
+      # Compute page_tempalte outside of profiler run, uninteresting to profiler
+      page_template = File.basename(template_name, File.extname(template_name))
+      compile_and_render(liquid, layout, assigns, page_template) 
+       
+    end
+  end
+
+
+  def run_profile
     RubyProf.measure_mode = RubyProf::WALL_TIME
 
     # Dup assigns because will make some changes to them

test/liquid/context_test.rb

@@ -254,12 +254,16 @@ class ContextTest < Test::Unit::TestCase
     @context['test'] = {'test' => [1,2,3,4,5]}
 
     assert_equal 1, @context['test.test[0]']
+  end
 
+  def test_recoursive_array_notation_for_hash
     @context['test'] = [{'test' => 'worked'}]
 
     assert_equal 'worked', @context['test[0].test']
   end
 
+
+
   def test_hash_to_array_transition
     @context['colors'] = {
       'Blue'    => ['003366','336699', '6699CC', '99CCFF'],
@@ -315,7 +319,7 @@ class ContextTest < Test::Unit::TestCase
     @context['nested'] = {'var' => 'tags'}
     @context['products'] = {'count' => 5, 'tags' => ['deepsnow', 'freestyle'] }
 
-    assert_equal 'deepsnow', @context['products[var].first']
+    #assert_equal 'deepsnow', @context['products[var].first']
     assert_equal 'freestyle', @context['products[nested.var].last']
   end
 

test/liquid/parser_test.rb

@@ -0,0 +1,89 @@
+require 'test_helper'
+
+
+class ParserTest < Test::Unit::TestCase
+  include Liquid
+
+
+  def test_strings
+    assert_equal [[:id, "string"]], Parser.parse('"string"')
+    assert_equal [[:id, "string"]], Parser.parse('\'string\'')
+  end
+
+  def test_integer
+    assert_equal [[:id, 1]], Parser.parse('1')
+    assert_equal [[:id, 100001]], Parser.parse('100001')
+  end
+
+  def test_float
+    assert_equal [[:id, 1.1]], Parser.parse('1.1')
+    assert_equal [[:id, 1.55435]], Parser.parse('1.55435')
+  end
+
+  def test_null
+    assert_equal [[:id, nil]], Parser.parse('null')
+    assert_equal [[:id, nil]], Parser.parse('nil')
+  end
+
+  def test_bool
+    assert_equal [[:id, true]], Parser.parse('true')
+    assert_equal [[:id, false]], Parser.parse('false')
+  end
+
+  def test_ranges
+    assert_equal [[:id, 1], [:id, 5], [:range, nil]], Parser.parse('(1..5)')
+    assert_equal [[:id, 100], [:id, 500], [:range, nil]], Parser.parse('(100..500)')    
+  end
+
+  def test_ranges_with_lookups
+    assert_equal [[:id, 1], [:id, "test"], [:lookup, nil], [:range, nil]], Parser.parse('(1..test)')
+  end
+
+  def test_lookups
+    assert_equal [[:id, "variable"], [:lookup, nil]], Parser.parse('variable')
+    assert_equal [[:id, "underscored_variable"], [:lookup, nil]], Parser.parse('underscored_variable')
+  end
+
+  def test_global_hash
+    assert_equal [[:id, true], [:lookup, nil]], Parser.parse('[true]')
+
+    assert_equal [[:id, "string"], [:lookup, nil]], Parser.parse('["string"]')
+    assert_equal [[:id, 5.55], [:lookup, nil]], Parser.parse('[5.55]')
+    assert_equal [[:id, 0], [:lookup, nil]], Parser.parse('[0]')
+    assert_equal [[:id, "variable"], [:lookup, nil], [:lookup, nil]], Parser.parse('[variable]')
+  end
+
+  def test_descent
+    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "variable2"], [:call, nil]], Parser.parse('variable1.variable2')
+    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "variable2"], [:call, nil], [:id, "variable3"], [:call, nil]], Parser.parse('variable1.variable2.variable3')
+  end
+
+  def test_descent_hash
+    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "variable2"], [:call, nil]], Parser.parse('variable1["variable2"]')  
+    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "variable2"], [:lookup, nil], [:call, nil]], Parser.parse('variable1[variable2]')
+  end
+
+  def test_buildin 
+    assert_equal [[:id, "first"], [:lookup, nil]], Parser.parse('first')  
+
+    assert_equal [[:id, "var"], [:lookup, nil], [:buildin, "first"]], Parser.parse('var.first')
+    assert_equal [[:id, "var"], [:lookup, nil], [:buildin, "last"]], Parser.parse('var.last')
+    assert_equal [[:id, "var"], [:lookup, nil], [:buildin, "size"]], Parser.parse('var.size')
+
+  end
+
+  def test_descent_hash_descent
+    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "test1"], [:lookup, nil], [:id, "test2"], [:call, nil], [:call, nil]], 
+      Parser.parse('variable1[test1.test2]'), "resolove: variable1[test1.test2]"
+
+    # assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "test1"], [:lookup, nil], [:id, "test2"], [:call, nil], [:call, nil]], 
+    #   Parser.parse('variable1[test1["test2"]]'), 'resolove: variable1[test1["test2"]]'
+
+    # assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "test1"], [:lookup, nil], [:id, "test2"], [:lookup, nil], [:call, nil], [:call, nil]], 
+    #   Parser.parse('variable1[test1[test2]]'), "resolove: variable1[test1[test2]]"
+
+  end
+
+
+
+end

test/liquid/parsing_quirks_test.rb

@@ -39,7 +39,7 @@ class ParsingQuirksTest < Test::Unit::TestCase
 
   def test_meaningless_parens
     assigns = {'b' => 'bar', 'c' => 'baz'}
-    markup = "a == 'foo' or (b == 'bar' and c == 'baz') or false"
+    markup = "a == 'foo' or b == 'bar' and c == 'baz' or false"
     assert_template_result(' YES ',"{% if #{markup} %} YES {% endif %}", assigns)
   end