Implement tokenization in a C extension.

.gitignore

@@ -6,3 +6,7 @@ pkg
 .rvmrc
 .ruby-version
 Gemfile.lock
+/ext/liquid/Makefile
+*.o
+*.bundle
+/tmp

.travis.yml

@@ -1,8 +1,7 @@
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1
-  - 2.1.1
+  - 2.1.0
   - jruby-19mode
   - jruby-head
   - rbx-19mode
@@ -11,7 +10,7 @@ matrix:
     - rvm: rbx-19mode
     - rvm: jruby-head
 
-script: "gem build liquid.gemspec && gem install liquid-3.0.0.gem"
+script: "rake test"
 
 notifications:
   disable: true

History.md

@@ -3,10 +3,6 @@
 ## 3.0.0 / not yet released / branch "master"
 
 * ...
-* Add error messages for missing variables when :strict, see #352 [Daniel Gaiottino]
-* Properly set context rethrow_errors on render! #349 [Thierry Joyal, tjoyal]
-* Fix broken rendering of variables which are equal to false, see #345 [Florian Weingarten, fw42]
-* Remove ActionView template handler [Dylan Thacker-Smith, dylanahsmith]
 * Freeze lots of string literals for new Ruby 2.1 optimization, see #297 [Florian Weingarten, fw42]
 * Allow newlines in tags and variables, see #324 [Dylan Thacker-Smith, dylanahsmith]
 * Tag#parse is called after initialize, which now takes options instead of tokens as the 3rd argument. See #321 [Dylan Thacker-Smith, dylanahsmith]

README.md

@@ -1,6 +1,4 @@
 [![Build Status](https://secure.travis-ci.org/Shopify/liquid.png?branch=master)](http://travis-ci.org/Shopify/liquid)
-[![Inline docs](http://inch-ci.org/github/Shopify/liquid.png)](http://inch-ci.org/github/Shopify/liquid)
-
 # Liquid template engine
 
 * [Contributing guidelines](CONTRIBUTING.md)

Rakefile

@@ -8,7 +8,7 @@ task :default => 'test'
 desc 'run test suite with default parser'
 Rake::TestTask.new(:base_test) do |t|
   t.libs << '.' << 'lib' << 'test'
-  t.test_files = FileList['test/{integration,unit}/**/*_test.rb']
+  t.test_files = FileList['test/liquid/**/*_test.rb']
   t.verbose = false
 end
 
@@ -75,3 +75,11 @@ desc "Run example"
 task :example do
   ruby "-w -d -Ilib example/server/server.rb"
 end
+
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
+  require 'rake/extensiontask'
+  Rake::ExtensionTask.new "liquid" do |ext|
+    ext.lib_dir = "lib/liquid"
+  end
+  Rake::Task[:test].prerequisites << :compile
+end

ext/liquid/extconf.rb

@@ -0,0 +1,4 @@
+require 'mkmf'
+$CFLAGS << ' -Wall -Werror'
+$warnflags.gsub!(/-Wdeclaration-after-statement/, "")
+create_makefile("liquid/liquid")

ext/liquid/liquid.c

@@ -0,0 +1,9 @@
+#include "liquid.h"
+
+VALUE mLiquid;
+
+void Init_liquid(void)
+{
+    mLiquid = rb_define_module("Liquid");
+    init_liquid_tokenizer();
+}

ext/liquid/liquid.h

@@ -0,0 +1,11 @@
+#ifndef LIQUID_H
+#define LIQUID_H
+
+#include <ruby.h>
+#include <stdbool.h>
+
+#include "tokenizer.h"
+
+extern VALUE mLiquid;
+
+#endif

ext/liquid/tokenizer.c

@@ -0,0 +1,137 @@
+#include "liquid.h"
+
+VALUE cLiquidTokenizer;
+
+static void tokenizer_mark(void *ptr) {
+    tokenizer_t *tokenizer = ptr;
+    rb_gc_mark(tokenizer->source);
+}
+
+static void tokenizer_free(void *ptr)
+{
+    tokenizer_t *tokenizer = ptr;
+    xfree(tokenizer);
+}
+
+static size_t tokenizer_memsize(const void *ptr)
+{
+    return ptr ? sizeof(tokenizer_t) : 0;
+}
+
+const rb_data_type_t tokenizer_data_type = {
+    "liquid_tokenizer",
+    {tokenizer_mark, tokenizer_free, tokenizer_memsize,},
+#ifdef RUBY_TYPED_FREE_IMMEDIATELY
+    NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY
+#endif
+};
+
+static VALUE tokenizer_allocate(VALUE klass)
+{
+    VALUE obj;
+    tokenizer_t *tokenizer;
+
+    obj = TypedData_Make_Struct(klass, tokenizer_t, &tokenizer_data_type, tokenizer);
+    tokenizer->source = Qnil;
+    return obj;
+}
+
+static VALUE tokenizer_initialize_method(VALUE self, VALUE source)
+{
+    tokenizer_t *tokenizer;
+
+    Check_Type(source, T_STRING);
+    Tokenizer_Get_Struct(self, tokenizer);
+    source = rb_str_dup_frozen(source);
+    tokenizer->source = source;
+    tokenizer->cursor = RSTRING_PTR(source);
+    tokenizer->length = RSTRING_LEN(source);
+    return Qnil;
+}
+
+void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
+{
+    if (tokenizer->length <= 0) {
+        memset(token, 0, sizeof(*token));
+        return;
+    }
+
+    const char *cursor = tokenizer->cursor;
+    const char *last = cursor + tokenizer->length - 1;
+
+    token->str = cursor;
+    token->type = TOKEN_STRING;
+
+    while (cursor < last) {
+        if (*cursor++ != '{')
+            continue;
+
+        char c = *cursor++;
+        if (c != '%' && c != '{')
+            continue;
+        if (cursor - tokenizer->cursor > 2) {
+            token->type = TOKEN_STRING;
+            cursor -= 2;
+            goto found;
+        }
+        token->type = TOKEN_INVALID;
+        if (c == '%') {
+            while (cursor < last) {
+                if (*cursor++ != '%')
+                    continue;
+                c = *cursor++;
+                while (c == '%' && cursor <= last)
+                    c = *cursor++;
+                if (c != '}')
+                    continue;
+                token->type = TOKEN_TAG;
+                goto found;
+            }
+            // unterminated tag
+            cursor = tokenizer->cursor + 2;
+            goto found;
+        } else {
+            while (cursor < last) {
+                if (*cursor++ != '}')
+                    continue;
+                if (*cursor++ != '}') {
+                    // variable incomplete end, used to end raw tags
+                    cursor--;
+                    goto found;
+                }
+                token->type = TOKEN_VARIABLE;
+                goto found;
+            }
+            // unterminated variable
+            cursor = tokenizer->cursor + 2;
+            goto found;
+        }
+    }
+    cursor = last + 1;
+found:
+    token->length = cursor - tokenizer->cursor;
+    tokenizer->cursor += token->length;
+    tokenizer->length -= token->length;
+}
+
+static VALUE tokenizer_next_method(VALUE self)
+{
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(self, tokenizer);
+
+    token_t token;
+    tokenizer_next(tokenizer, &token);
+    if (token.type == TOKEN_NONE)
+        return Qnil;
+
+    return rb_str_new(token.str, token.length);
+}
+
+void init_liquid_tokenizer()
+{
+    cLiquidTokenizer = rb_define_class_under(mLiquid, "Tokenizer", rb_cObject);
+    rb_define_alloc_func(cLiquidTokenizer, tokenizer_allocate);
+    rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 1);
+    rb_define_method(cLiquidTokenizer, "next", tokenizer_next_method, 0);
+    rb_define_alias(cLiquidTokenizer, "shift", "next");
+}

ext/liquid/tokenizer.h

@@ -0,0 +1,31 @@
+#ifndef LIQUID_TOKENIZER_H
+#define LIQUID_TOKENIZER_H
+
+enum token_type {
+    TOKEN_NONE,
+    TOKEN_INVALID,
+    TOKEN_STRING,
+    TOKEN_TAG,
+    TOKEN_VARIABLE
+};
+
+typedef struct token {
+    enum token_type type;
+    const char *str;
+    long length;
+} token_t;
+
+typedef struct tokenizer {
+    VALUE source;
+    const char *cursor;
+    long length;
+} tokenizer_t;
+
+extern VALUE cLiquidTokenizer;
+extern const rb_data_type_t tokenizer_data_type;
+#define Tokenizer_Get_Struct(obj, sval) TypedData_Get_Struct(obj, tokenizer_t, &tokenizer_data_type, sval)
+
+void init_liquid_tokenizer();
+void tokenizer_next(tokenizer_t *tokenizer, token_t *token);
+
+#endif

init.rb

@@ -0,0 +1,8 @@
+require 'liquid'
+require 'extras/liquid_view'
+
+if defined? ActionView::Template and ActionView::Template.respond_to? :register_template_handler
+  ActionView::Template
+else
+  ActionView::Base
+end.register_template_handler(:liquid, LiquidView)

lib/extras/liquid_view.rb

@@ -0,0 +1,51 @@
+# LiquidView is a action view extension class. You can register it with rails
+# and use liquid as an template system for .liquid files
+#
+# Example
+#
+#   ActionView::Base::register_template_handler :liquid, LiquidView
+class LiquidView
+  PROTECTED_ASSIGNS = %w( template_root response _session template_class action_name request_origin session template
+                          _response url _request _cookies variables_added _flash params _headers request cookies
+                          ignore_missing_templates flash _params logger before_filter_chain_aborted headers )
+  PROTECTED_INSTANCE_VARIABLES = %w( @_request @controller @_first_render @_memoized__pick_template @view_paths
+                                     @helpers @assigns_added @template @_render_stack @template_format @assigns )
+
+  def self.call(template)
+    "LiquidView.new(self).render(template, local_assigns)"
+  end
+
+  def initialize(view)
+    @view = view
+  end
+
+  def render(template, local_assigns = nil)
+    @view.controller.headers["Content-Type"] ||= 'text/html; charset=utf-8'
+
+    # Rails 2.2 Template has source, but not locals
+    if template.respond_to?(:source) && !template.respond_to?(:locals)
+      assigns = (@view.instance_variables - PROTECTED_INSTANCE_VARIABLES).inject({}) do |hash, ivar|
+                  hash[ivar[1..-1]] = @view.instance_variable_get(ivar)
+                  hash
+                end
+    else
+      assigns = @view.assigns.reject{ |k,v| PROTECTED_ASSIGNS.include?(k) }
+    end
+
+    source = template.respond_to?(:source) ? template.source : template
+    local_assigns = (template.respond_to?(:locals) ? template.locals : local_assigns) || {}
+
+    if content_for_layout = @view.instance_variable_get("@content_for_layout")
+      assigns['content_for_layout'] = content_for_layout
+    end
+    assigns.merge!(local_assigns.stringify_keys)
+
+    liquid = Liquid::Template.parse(source)
+    liquid.render(assigns, :filters => [@view.controller.master_helper_module], :registers => {:action_view => @view, :controller => @view.controller})
+  end
+
+  def compilable?
+    false
+  end
+
+end

lib/liquid.rb

@@ -30,13 +30,9 @@ module Liquid
   VariableSegment             = /[\w\-]/
   VariableStart               = /\{\{/
   VariableEnd                 = /\}\}/
-  VariableIncompleteEnd       = /\}\}?/
   QuotedString                = /"[^"]*"|'[^']*'/
   QuotedFragment              = /#{QuotedString}|(?:[^\s,\|'"]|#{QuotedString})+/o
   TagAttributes               = /(\w+)\s*\:\s*(#{QuotedFragment})/o
-  AnyStartingTag              = /\{\{|\{\%/
-  PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om
-  TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/om
   VariableParser              = /\[[^\]]+\]|#{VariableSegment}+\??/o
 end
 
@@ -64,3 +60,9 @@ require 'liquid/utils'
 # Load all the tags of the standard library
 #
 Dir[File.dirname(__FILE__) + '/liquid/tags/*.rb'].each { |f| require f }
+
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
+  require 'liquid/liquid'
+else
+  require 'liquid/tokenizer'
+end

lib/liquid/context.rb

@@ -15,8 +15,6 @@ module Liquid
   class Context
     attr_reader :scopes, :errors, :registers, :environments, :resource_limits
 
-    attr_accessor :rethrow_errors
-
     def initialize(environments = {}, outer_scope = {}, registers = {}, rethrow_errors = false, resource_limits = {})
       @environments    = [environments].flatten
       @scopes          = [(outer_scope || {})]
@@ -196,8 +194,7 @@ module Liquid
 
         if scope.nil?
           @environments.each do |e|
-            variable = lookup_and_evaluate(e, key)
-            unless variable.nil?
+            if variable = lookup_and_evaluate(e, key)
               scope = e
               break
             end
@@ -205,7 +202,6 @@ module Liquid
         end
 
         scope     ||= @environments.last || @scopes.last
-        handle_not_found(key) unless scope.has_key?(key)
         variable  ||= lookup_and_evaluate(scope, key)
 
         variable = variable.to_liquid
@@ -255,7 +251,6 @@ module Liquid
               # No key was present with the desired value and it wasn't one of the directly supported
               # keywords either. The only thing we got left is to return nil
             else
-              handle_not_found(markup)
               return nil
             end
 
@@ -285,10 +280,6 @@ module Liquid
           end
         end
       end # squash_instance_assigns_with_environments
-
-      def handle_not_found(variable)
-        @errors << "Variable {{#{variable}}} not found" if Template.error_mode == :strict
-      end
   end # Context
 
 end # Liquid

lib/liquid/standardfilters.rb

@@ -162,7 +162,7 @@ module Liquid
       input.to_s.gsub(/\n/, "<br />\n".freeze)
     end
 
-    # Reformat a date using Ruby's core Time#strftime( string ) -> string
+    # Reformat a date
     #
     #   %a - The abbreviated weekday name (``Sun'')
     #   %A - The  full  weekday  name (``Sunday'')
@@ -176,7 +176,6 @@ module Liquid
     #   %m - Month of the year (01..12)
     #   %M - Minute of the hour (00..59)
     #   %p - Meridian indicator (``AM''  or  ``PM'')
-    #   %s - Number of seconds since 1970-01-01 00:00:00 UTC.
     #   %S - Second of the minute (00..60)
     #   %U - Week  number  of the current year,
     #           starting with the first Sunday as the first
@@ -191,14 +190,34 @@ module Liquid
     #   %Y - Year with century
     #   %Z - Time zone name
     #   %% - Literal ``%'' character
-    #
-    #   See also: http://www.ruby-doc.org/core/Time.html#method-i-strftime
     def date(input, format)
-      return input if format.to_s.empty?
 
-      return input unless date = to_date(input)
+      if format.to_s.empty?
+        return input.to_s
+      end
 
-      date.strftime(format.to_s)
+      if ((input.is_a?(String) && !/\A\d+\z/.match(input.to_s).nil?) || input.is_a?(Integer)) && input.to_i > 0
+        input = Time.at(input.to_i)
+      end
+
+      date = if input.is_a?(String)
+        case input.downcase
+        when 'now'.freeze, 'today'.freeze
+          Time.now
+        else
+          Time.parse(input)
+        end
+      else
+        input
+      end
+
+      if date.respond_to?(:strftime)
+        date.strftime(format.to_s)
+      else
+        input
+      end
+    rescue
+      input
     end
 
     # Get the first element of the passed in array
@@ -243,21 +262,6 @@ module Liquid
       apply_operation(input, operand, :%)
     end
 
-    def round(input, n = 0)
-      result = to_number(input).round(to_number(n))
-      result = result.to_f if result.is_a?(BigDecimal)
-      result = result.to_i if n == 0
-      result
-    end
-
-    def ceil(input)
-      to_number(input).ceil.to_i
-    end
-
-    def floor(input)
-      to_number(input).floor.to_i
-    end
-
     def default(input, default_value = "".freeze)
       is_blank = input.respond_to?(:empty?) ? input.empty? : !input
       is_blank ? default_value : input
@@ -289,23 +293,6 @@ module Liquid
       end
     end
 
-    def to_date(obj)
-      return obj if obj.respond_to?(:strftime)
-
-      case obj
-      when 'now'.freeze, 'today'.freeze
-        Time.now
-      when /\A\d+\z/, Integer
-        Time.at(obj.to_i)
-      when String
-        Time.parse(obj)
-      else
-        nil
-      end
-    rescue ArgumentError
-      nil
-    end
-
     def apply_operation(input, operand, operation)
       result = to_number(input).send(operation, to_number(operand))
       result.is_a?(BigDecimal) ? result.to_f : result

lib/liquid/template.rb

@@ -22,12 +22,6 @@ module Liquid
     @@file_system = BlankFileSystem.new
 
     class << self
-      # Sets how strict the parser should be.
-      # :lax acts like liquid 2.5 and silently ignores malformed tags in most cases.
-      # :warn is the default and will give deprecation warnings when invalid syntax is used.
-      # :strict will enforce correct syntax.
-      attr_writer :error_mode
-
       def file_system
         @@file_system
       end
@@ -44,6 +38,14 @@ module Liquid
         @tags ||= {}
       end
 
+      # Sets how strict the parser should be.
+      # :lax acts like liquid 2.5 and silently ignores malformed tags in most cases.
+      # :warn is the default and will give deprecation warnings when invalid syntax is used.
+      # :strict will enforce correct syntax.
+      def error_mode=(mode)
+        @error_mode = mode
+      end
+
       def error_mode
         @error_mode || :lax
       end
@@ -58,6 +60,7 @@ module Liquid
       def parse(source, options = {})
         template = Template.new
         template.parse(source, options)
+        template
       end
     end
 
@@ -111,9 +114,7 @@ module Liquid
 
       context = case args.first
       when Liquid::Context
-        c = args.shift
-        c.rethrow_errors = true if @rethrow_errors
-        c
+        args.shift
       when Liquid::Drop
         drop = args.shift
         drop.context = Context.new([drop, assigns], instance_assigns, registers, @rethrow_errors, @resource_limits)
@@ -156,22 +157,14 @@ module Liquid
     end
 
     def render!(*args)
-      @rethrow_errors = true
-      render(*args)
+      @rethrow_errors = true; render(*args)
     end
 
     private
 
-    # Uses the <tt>Liquid::TemplateParser</tt> regexp to tokenize the passed source
     def tokenize(source)
       source = source.source if source.respond_to?(:source)
-      return [] if source.to_s.empty?
-      tokens = source.split(TemplateParser)
-
-      # removes the rogue empty element at the beginning of the array
-      tokens.shift if tokens[0] and tokens[0].empty?
-
-      tokens
+      Tokenizer.new(source.to_s)
     end
 
   end

lib/liquid/tokenizer.rb

@@ -0,0 +1,20 @@
+module Liquid
+  class Tokenizer
+    VariableIncompleteEnd = /\}\}?/
+    AnyStartingTag        = /\{\{|\{\%/
+    PartialTemplateParser = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om
+    TemplateParser        = /(#{PartialTemplateParser}|#{AnyStartingTag})/om
+
+    def initialize(source)
+      @tokens = source.split(TemplateParser)
+
+      # removes the rogue empty element at the beginning of the array
+      @tokens.shift if @tokens[0] && @tokens[0].empty?
+    end
+
+    def next
+      @tokens.shift
+    end
+    alias_method :shift, :next
+  end
+end

liquid.gemspec

@@ -18,12 +18,17 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = ">= 1.3.7"
 
   s.test_files  = Dir.glob("{test}/**/*")
-  s.files       = Dir.glob("{lib}/**/*") + %w(MIT-LICENSE README.md)
+  s.files       = Dir.glob("{lib,ext}/**/*") + %w(MIT-LICENSE README.md)
 
   s.extra_rdoc_files  = ["History.md", "README.md"]
 
   s.require_path = "lib"
 
-  s.add_development_dependency 'stackprof' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0")
   s.add_development_dependency 'rake'
+  s.add_development_dependency 'activesupport'
+  if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
+    s.extensions  = ['ext/liquid/extconf.rb']
+    s.add_development_dependency 'rake-compiler'
+    s.add_development_dependency 'stackprof' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0")
+  end
 end

performance/benchmark.rb

@@ -1,3 +1,4 @@
+require 'rubygems'
 require 'benchmark'
 require File.dirname(__FILE__) + '/theme_runner'
 

performance/shopify/database.rb

@@ -1,6 +1,6 @@
 require 'yaml'
-
 module Database
+
   # Load the standard vision toolkit database and re-arrage it to be simply exportable
   # to liquid as assigns. All this is based on Shopify
   def self.tables

performance/shopify/json_filter.rb

@@ -1,9 +1,7 @@
-require 'json'
-
 module JsonFilter
 
   def json(object)
-    JSON.dump(object.reject {|k,v| k == "collections" })
+    object.reject {|k,v| k == "collections" }.to_json
   end
 
 end

performance/theme_runner.rb

@@ -6,6 +6,11 @@
 # Shopify which is likely the biggest user of liquid in the world which something to the tune of several
 # million Template#render calls a day.
 
+require 'rubygems'
+require 'active_support'
+require 'active_support/json'
+require 'yaml'
+require 'digest/md5'
 require File.dirname(__FILE__) + '/shopify/liquid'
 require File.dirname(__FILE__) + '/shopify/database.rb'
 
@@ -76,3 +81,6 @@ class ThemeRunner
     end
   end
 end
+
+
+

test/integration/context_test.rb

@@ -1,23 +0,0 @@
-require 'test_helper'
-
-class ContextTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_override_global_filter
-    global = Module.new do
-      def notice(output)
-        "Global #{output}"
-      end
-    end
-
-    local = Module.new do
-      def notice(output)
-        "Local #{output}"
-      end
-    end
-
-    Template.register_filter(global)
-    assert_equal 'Global test', Template.parse("{{'test' | notice }}").render!
-    assert_equal 'Local test', Template.parse("{{'test' | notice }}").render!({}, :filters => [local])
-  end
-end

test/integration/variable_test.rb

@@ -1,72 +0,0 @@
-require 'test_helper'
-
-class VariableTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_simple_variable
-    template = Template.parse(%|{{test}}|)
-    assert_equal 'worked', template.render!('test' => 'worked')
-    assert_equal 'worked wonderfully', template.render!('test' => 'worked wonderfully')
-  end
-
-  def test_simple_with_whitespaces
-    template = Template.parse(%|  {{ test }}  |)
-    assert_equal '  worked  ', template.render!('test' => 'worked')
-    assert_equal '  worked wonderfully  ', template.render!('test' => 'worked wonderfully')
-  end
-
-  def test_ignore_unknown
-    template = Template.parse(%|{{ test }}|)
-    assert_equal '', template.render!
-  end
-
-  def test_hash_scoping
-    template = Template.parse(%|{{ test.test }}|)
-    assert_equal 'worked', template.render!('test' => {'test' => 'worked'})
-  end
-
-  def test_false_renders_as_false
-    assert_equal 'false', Template.parse("{{ foo }}").render!('foo' => false)
-  end
-
-  def test_preset_assigns
-    template = Template.parse(%|{{ test }}|)
-    template.assigns['test'] = 'worked'
-    assert_equal 'worked', template.render!
-  end
-
-  def test_reuse_parsed_template
-    template = Template.parse(%|{{ greeting }} {{ name }}|)
-    template.assigns['greeting'] = 'Goodbye'
-    assert_equal 'Hello Tobi', template.render!('greeting' => 'Hello', 'name' => 'Tobi')
-    assert_equal 'Hello ', template.render!('greeting' => 'Hello', 'unknown' => 'Tobi')
-    assert_equal 'Hello Brian', template.render!('greeting' => 'Hello', 'name' => 'Brian')
-    assert_equal 'Goodbye Brian', template.render!('name' => 'Brian')
-    assert_equal({'greeting'=>'Goodbye'}, template.assigns)
-  end
-
-  def test_assigns_not_polluted_from_template
-    template = Template.parse(%|{{ test }}{% assign test = 'bar' %}{{ test }}|)
-    template.assigns['test'] = 'baz'
-    assert_equal 'bazbar', template.render!
-    assert_equal 'bazbar', template.render!
-    assert_equal 'foobar', template.render!('test' => 'foo')
-    assert_equal 'bazbar', template.render!
-  end
-
-  def test_hash_with_default_proc
-    template = Template.parse(%|Hello {{ test }}|)
-    assigns = Hash.new { |h,k| raise "Unknown variable '#{k}'" }
-    assigns['test'] = 'Tobi'
-    assert_equal 'Hello Tobi', template.render!(assigns)
-    assigns.delete('test')
-    e = assert_raises(RuntimeError) {
-      template.render!(assigns)
-    }
-    assert_equal "Unknown variable 'test'", e.message
-  end
-
-  def test_multiline_variable
-    assert_equal 'worked', Template.parse("{{\ntest\n}}").render!('test' => 'worked')
-  end
-end

test/liquid/block_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class BlockUnitTest < Test::Unit::TestCase
+class BlockTest < Test::Unit::TestCase
   include Liquid
 
   def test_blankspace

test/liquid/condition_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class ConditionUnitTest < Test::Unit::TestCase
+class ConditionTest < Test::Unit::TestCase
   include Liquid
 
   def test_basic_condition

test/liquid/context_test.rb

@@ -63,7 +63,7 @@ class ArrayLike
   end
 end
 
-class ContextUnitTest < Test::Unit::TestCase
+class ContextTest < Test::Unit::TestCase
   include Liquid
 
   def setup
@@ -162,6 +162,24 @@ class ContextUnitTest < Test::Unit::TestCase
 
   end
 
+  def test_override_global_filter
+    global = Module.new do
+      def notice(output)
+        "Global #{output}"
+      end
+    end
+
+    local = Module.new do
+      def notice(output)
+        "Local #{output}"
+      end
+    end
+
+    Template.register_filter(global)
+    assert_equal 'Global test', Template.parse("{{'test' | notice }}").render!
+    assert_equal 'Local test', Template.parse("{{'test' | notice }}").render!({}, :filters => [local])
+  end
+
   def test_only_intended_filters_make_it_there
 
     filter = Module.new do
@@ -457,22 +475,4 @@ class ContextUnitTest < Test::Unit::TestCase
     assert_kind_of CategoryDrop, @context['category']
     assert_equal @context, @context['category'].context
   end
-
-  def test_strict_variables_not_found
-    with_error_mode(:strict) do
-      @context['does_not_exist']
-      assert(@context.errors.length == 1)
-      assert_equal(@context.errors[0], 'Variable {{does_not_exist}} not found')
-    end
-  end
-
-  def test_strict_nested_variables_not_found
-    with_error_mode(:strict) do
-      @context['hash'] = {'this' => 'exists'}
-      @context['hash.does_not_exist']
-      assert(@context.errors.length == 1)
-      assert_equal(@context.errors[0], 'Variable {{hash.does_not_exist}} not found')
-    end
-  end
-
 end # ContextTest

test/liquid/file_system_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class FileSystemUnitTest < Test::Unit::TestCase
+class FileSystemTest < Test::Unit::TestCase
   include Liquid
 
   def test_default

test/liquid/i18n_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class I18nUnitTest < Test::Unit::TestCase
+class I18nTest < Test::Unit::TestCase
   include Liquid
 
   def setup

test/liquid/lexer_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class LexerUnitTest < Test::Unit::TestCase
+class LexerTest < Test::Unit::TestCase
   include Liquid
 
   def test_strings

test/liquid/module_ex_test.rb

@@ -36,7 +36,7 @@ class TestClassC::LiquidDropClass
   end
 end
 
-class ModuleExUnitTest < Test::Unit::TestCase
+class ModuleExTest < Test::Unit::TestCase
   include Liquid
 
   def setup

test/liquid/parser_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class ParserUnitTest < Test::Unit::TestCase
+class ParserTest < Test::Unit::TestCase
   include Liquid
 
   def test_consume

test/liquid/parsing_quirks_test.rb

@@ -3,9 +3,12 @@ require 'test_helper'
 class ParsingQuirksTest < Test::Unit::TestCase
   include Liquid
 
-  def test_parsing_css
-    text = " div { font-weight: bold; } "
-    assert_equal text, Template.parse(text).render!
+  def test_error_with_css
+    text = %| div { font-weight: bold; } |
+    template = Template.parse(text)
+
+    assert_equal text, template.render!
+    assert_equal [String], template.root.nodelist.collect {|i| i.class}
   end
 
   def test_raise_on_single_close_bracet

test/liquid/regexp_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class RegexpUnitTest < Test::Unit::TestCase
+class RegexpTest < Test::Unit::TestCase
   include Liquid
 
   def test_empty

test/liquid/security_test.rb

@@ -54,10 +54,10 @@ class SecurityTest < Test::Unit::TestCase
   def test_does_not_add_drop_methods_to_symbol_table
     current_symbols = Symbol.all_symbols
 
-    assigns = { 'drop' => Drop.new }
-    assert_equal "", Template.parse("{{ drop.custom_method_1 }}", assigns).render!
-    assert_equal "", Template.parse("{{ drop.custom_method_2 }}", assigns).render!
-    assert_equal "", Template.parse("{{ drop.custom_method_3 }}", assigns).render!
+    drop = Drop.new
+    drop.invoke_drop("custom_method_1")
+    drop.invoke_drop("custom_method_2")
+    drop.invoke_drop("custom_method_3")
 
     assert_equal [], (Symbol.all_symbols - current_symbols)
   end

test/liquid/standard_filter_test.rb

@@ -115,13 +115,6 @@ class StandardFiltersTest < Test::Unit::TestCase
     assert_equal [{"a" => 1}, {"a" => 2}, {"a" => 3}, {"a" => 4}], @filters.sort([{"a" => 4}, {"a" => 3}, {"a" => 1}, {"a" => 2}], "a")
   end
 
-  def test_numerical_vs_lexicographical_sort
-    assert_equal [2, 10], @filters.sort([10, 2])
-    assert_equal [{"a" => 2}, {"a" => 10}], @filters.sort([{"a" => 10}, {"a" => 2}], "a")
-    assert_equal ["10", "2"], @filters.sort(["10", "2"])
-    assert_equal [{"a" => "10"}, {"a" => "2"}], @filters.sort([{"a" => "10"}, {"a" => "2"}], "a")
-  end
-
   def test_reverse
     assert_equal [4,3,2,1], @filters.reverse([1,2,3,4])
   end
@@ -193,6 +186,7 @@ class StandardFiltersTest < Test::Unit::TestCase
     assert_equal "07/05/2006", @filters.date("1152098955", "%m/%d/%Y")
   end
 
+
   def test_first_last
     assert_equal 1, @filters.first([1,2,3])
     assert_equal 3, @filters.last([1,2,3])
@@ -273,22 +267,6 @@ class StandardFiltersTest < Test::Unit::TestCase
     assert_template_result "1", "{{ 3 | modulo:2 }}"
   end
 
-  def test_round
-    assert_template_result "5", "{{ input | round }}", 'input' => 4.6
-    assert_template_result "4", "{{ '4.3' | round }}"
-    assert_template_result "4.56", "{{ input | round: 2 }}", 'input' => 4.5612
-  end
-
-  def test_ceil
-    assert_template_result "5", "{{ input | ceil }}", 'input' => 4.6
-    assert_template_result "5", "{{ '4.3' | ceil }}"
-  end
-
-  def test_floor
-    assert_template_result "4", "{{ input | floor }}", 'input' => 4.6
-    assert_template_result "4", "{{ '4.3' | floor }}"
-  end
-
   def test_append
     assigns = {'a' => 'bc', 'b' => 'd' }
     assert_template_result('bcd',"{{ a | append: 'd'}}",assigns)

test/liquid/strainer_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class StrainerUnitTest < Test::Unit::TestCase
+class StrainerTest < Test::Unit::TestCase
   include Liquid
 
   module AccessScopeFilters

test/liquid/tags/case_tag_test.rb

@@ -1,10 +1,10 @@
 require 'test_helper'
 
-class CaseTagUnitTest < Test::Unit::TestCase
+class CaseTagTest < Test::Unit::TestCase
   include Liquid
 
   def test_case_nodelist
     template = Liquid::Template.parse('{% case var %}{% when true %}WHEN{% else %}ELSE{% endcase %}')
     assert_equal ['WHEN', 'ELSE'], template.root.nodelist[0].nodelist
   end
-end
+end # CaseTest

test/liquid/tags/for_tag_test.rb

@@ -1,11 +1,5 @@
 require 'test_helper'
 
-class ThingWithValue < Liquid::Drop
-  def value
-    3
-  end
-end
-
 class ForTagTest < Test::Unit::TestCase
   include Liquid
 
@@ -40,20 +34,6 @@ HERE
     assert_template_result(' 1  2  3 ','{%for item in (1..3) %} {{item}} {%endfor%}')
   end
 
-  def test_for_with_variable_range
-    assert_template_result(' 1  2  3 ','{%for item in (1..foobar) %} {{item}} {%endfor%}', "foobar" => 3)
-  end
-
-  def test_for_with_hash_value_range
-    foobar = { "value" => 3 }
-    assert_template_result(' 1  2  3 ','{%for item in (1..foobar.value) %} {{item}} {%endfor%}', "foobar" => foobar)
-  end
-
-  def test_for_with_drop_value_range
-    foobar = ThingWithValue.new
-    assert_template_result(' 1  2  3 ','{%for item in (1..foobar.value) %} {{item}} {%endfor%}', "foobar" => foobar)
-  end
-
   def test_for_with_variable
     assert_template_result(' 1  2  3 ','{%for item in array%} {{item}} {%endfor%}','array' => [1,2,3])
     assert_template_result('123','{%for item in array%}{{item}}{%endfor%}','array' => [1,2,3])
@@ -315,6 +295,16 @@ HERE
     assert_template_result(expected, template, assigns)
   end
 
+  def test_for_nodelist
+    template = Liquid::Template.parse('{% for item in items %}FOR{% endfor %}')
+    assert_equal ['FOR'], template.root.nodelist[0].nodelist
+  end
+
+  def test_for_else_nodelist
+    template = Liquid::Template.parse('{% for item in items %}FOR{% else %}ELSE{% endfor %}')
+    assert_equal ['FOR', 'ELSE'], template.root.nodelist[0].nodelist
+  end
+
   class LoaderDrop < Liquid::Drop
     attr_accessor :each_called, :load_slice_called
 

test/liquid/tags/if_else_tag_test.rb

@@ -158,9 +158,14 @@ class IfElseTagTest < Test::Unit::TestCase
                            %({% if 'gnomeslab-and-or-liquid' contains 'gnomeslab-and-or-liquid' %}yes{% endif %}))
   end
 
+  def test_if_nodelist
+    template = Liquid::Template.parse('{% if true %}IF{% else %}ELSE{% endif %}')
+    assert_equal ['IF', 'ELSE'], template.root.nodelist[0].nodelist
+  end
+
   def test_operators_are_whitelisted
     assert_raise(SyntaxError) do
       assert_template_result('', %({% if 1 or throw or or 1 %}yes{% endif %}))
     end
   end
-end
+end # IfElseTest

test/liquid/tags/standard_tag_test.rb

@@ -3,6 +3,12 @@ require 'test_helper'
 class StandardTagTest < Test::Unit::TestCase
   include Liquid
 
+  def test_tag
+    tag = Tag.parse('tag', [], [], {})
+    assert_equal 'liquid::tag', tag.name
+    assert_equal '', tag.render(Context.new)
+  end
+
   def test_no_transform
     assert_template_result('this text should come out of the template without change...',
                            'this text should come out of the template without change...')

test/liquid/template_test.rb

@@ -22,12 +22,6 @@ class SomethingWithLength
   liquid_methods :length
 end
 
-class ErroneousDrop < Liquid::Drop
-  def bad_method
-    raise 'ruby error in drop'
-  end
-end
-
 class TemplateTest < Test::Unit::TestCase
   include Liquid
 
@@ -144,13 +138,17 @@ class TemplateTest < Test::Unit::TestCase
     assert_equal 'haha', t.parse("{{baz}}").render!(drop)
   end
 
-  def test_render_bang_force_rethrow_errors_on_passed_context
-    context = Context.new({'drop' => ErroneousDrop.new})
-    t = Template.new.parse('{{ drop.bad_method }}')
+  def test_sets_default_localization_in_document
+    t = Template.new
+    t.parse('')
+    assert_instance_of I18n, t.root.options[:locale]
+  end
 
-    e = assert_raises RuntimeError do
-      t.render!(context)
-    end
-    assert_equal 'ruby error in drop', e.message
+  def test_sets_default_localization_in_context_with_quick_initialization
+    t = Template.new
+    t.parse('{{foo}}', :locale => I18n.new(fixture("en_locale.yml")))
+
+    assert_instance_of I18n, t.root.options[:locale]
+    assert_equal fixture("en_locale.yml"), t.root.options[:locale].path
   end
 end

test/liquid/tokenizer_test.rb

@@ -24,6 +24,11 @@ class TokenizerTest < Test::Unit::TestCase
   private
 
   def tokenize(source)
-    Liquid::Template.new.send(:tokenize, source)
+    tokenizer = Liquid::Tokenizer.new(source)
+    tokens = []
+    while token = tokenizer.next
+      tokens << token
+    end
+    tokens
   end
 end

test/liquid/variable_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class VariableUnitTest < Test::Unit::TestCase
+class VariableTest < Test::Unit::TestCase
   include Liquid
 
   def test_variable
@@ -134,3 +134,71 @@ class VariableUnitTest < Test::Unit::TestCase
     end
   end
 end
+
+
+class VariableResolutionTest < Test::Unit::TestCase
+  include Liquid
+
+  def test_simple_variable
+    template = Template.parse(%|{{test}}|)
+    assert_equal 'worked', template.render!('test' => 'worked')
+    assert_equal 'worked wonderfully', template.render!('test' => 'worked wonderfully')
+  end
+
+  def test_simple_with_whitespaces
+    template = Template.parse(%|  {{ test }}  |)
+    assert_equal '  worked  ', template.render!('test' => 'worked')
+    assert_equal '  worked wonderfully  ', template.render!('test' => 'worked wonderfully')
+  end
+
+  def test_ignore_unknown
+    template = Template.parse(%|{{ test }}|)
+    assert_equal '', template.render!
+  end
+
+  def test_hash_scoping
+    template = Template.parse(%|{{ test.test }}|)
+    assert_equal 'worked', template.render!('test' => {'test' => 'worked'})
+  end
+
+  def test_preset_assigns
+    template = Template.parse(%|{{ test }}|)
+    template.assigns['test'] = 'worked'
+    assert_equal 'worked', template.render!
+  end
+
+  def test_reuse_parsed_template
+    template = Template.parse(%|{{ greeting }} {{ name }}|)
+    template.assigns['greeting'] = 'Goodbye'
+    assert_equal 'Hello Tobi', template.render!('greeting' => 'Hello', 'name' => 'Tobi')
+    assert_equal 'Hello ', template.render!('greeting' => 'Hello', 'unknown' => 'Tobi')
+    assert_equal 'Hello Brian', template.render!('greeting' => 'Hello', 'name' => 'Brian')
+    assert_equal 'Goodbye Brian', template.render!('name' => 'Brian')
+    assert_equal({'greeting'=>'Goodbye'}, template.assigns)
+  end
+
+  def test_assigns_not_polluted_from_template
+    template = Template.parse(%|{{ test }}{% assign test = 'bar' %}{{ test }}|)
+    template.assigns['test'] = 'baz'
+    assert_equal 'bazbar', template.render!
+    assert_equal 'bazbar', template.render!
+    assert_equal 'foobar', template.render!('test' => 'foo')
+    assert_equal 'bazbar', template.render!
+  end
+
+  def test_hash_with_default_proc
+    template = Template.parse(%|Hello {{ test }}|)
+    assigns = Hash.new { |h,k| raise "Unknown variable '#{k}'" }
+    assigns['test'] = 'Tobi'
+    assert_equal 'Hello Tobi', template.render!(assigns)
+    assigns.delete('test')
+    e = assert_raises(RuntimeError) {
+      template.render!(assigns)
+    }
+    assert_equal "Unknown variable 'test'", e.message
+  end
+
+  def test_multiline_variable
+    assert_equal 'worked', Template.parse("{{\ntest\n}}").render!('test' => 'worked')
+  end
+end # VariableTest

test/unit/tag_unit_test.rb

@@ -1,11 +0,0 @@
-require 'test_helper'
-
-class TagUnitTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_tag
-    tag = Tag.parse('tag', [], [], {})
-    assert_equal 'liquid::tag', tag.name
-    assert_equal '', tag.render(Context.new)
-  end
-end

test/unit/tags/for_tag_unit_test.rb

@@ -1,13 +0,0 @@
-require 'test_helper'
-
-class ForTagUnitTest < Test::Unit::TestCase
-  def test_for_nodelist
-    template = Liquid::Template.parse('{% for item in items %}FOR{% endfor %}')
-    assert_equal ['FOR'], template.root.nodelist[0].nodelist
-  end
-
-  def test_for_else_nodelist
-    template = Liquid::Template.parse('{% for item in items %}FOR{% else %}ELSE{% endfor %}')
-    assert_equal ['FOR', 'ELSE'], template.root.nodelist[0].nodelist
-  end
-end

test/unit/tags/if_tag_unit_test.rb

@@ -1,8 +0,0 @@
-require 'test_helper'
-
-class IfTagUnitTest < Test::Unit::TestCase
-  def test_if_nodelist
-    template = Liquid::Template.parse('{% if true %}IF{% else %}ELSE{% endif %}')
-    assert_equal ['IF', 'ELSE'], template.root.nodelist[0].nodelist
-  end
-end

test/unit/template_unit_test.rb

@@ -1,19 +0,0 @@
-require 'test_helper'
-
-class TemplateUnitTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_sets_default_localization_in_document
-    t = Template.new
-    t.parse('')
-    assert_instance_of I18n, t.root.options[:locale]
-  end
-
-  def test_sets_default_localization_in_context_with_quick_initialization
-    t = Template.new
-    t.parse('{{foo}}', :locale => I18n.new(fixture("en_locale.yml")))
-
-    assert_instance_of I18n, t.root.options[:locale]
-    assert_equal fixture("en_locale.yml"), t.root.options[:locale].path
-  end
-end