Implement tokenization in a C extension.

.gitignore

@@ -6,3 +6,7 @@ pkg
 .rvmrc
 .ruby-version
 Gemfile.lock
+/ext/liquid/Makefile
+*.o
+*.bundle
+/tmp

.travis.yml

@@ -1,13 +1,13 @@
 rvm:
-  - 1.9
-  - 2.0
-  - 2.1
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
   - jruby-19mode
   - jruby-head
-  - rbx-2
+  - rbx-19mode
 matrix:
   allow_failures:
-    - rvm: rbx-2
+    - rvm: rbx-19mode
     - rvm: jruby-head
 
 script: "rake test"

Gemfile

@@ -1,8 +1,3 @@
 source 'https://rubygems.org'
 
 gemspec
-gem 'stackprof', platforms: :mri_21
-
-group :test do
-  gem 'spy', '0.4.1'
-end

History.md

@@ -3,12 +3,6 @@
 ## 3.0.0 / not yet released / branch "master"
 
 * ...
-* Add exception_handler feature, see #397 and #254 [Bogdan Gusiev, bogdan and Florian Weingarten, fw42]
-* Optimize variable parsing to avoid repeated regex evaluation during template rendering #383 [Jason Hiltz-Laforge, jasonhl]
-* Optimize checking for block interrupts to reduce object allocation #380 [Jason Hiltz-Laforge, jasonhl] 
-* Properly set context rethrow_errors on render! #349 [Thierry Joyal, tjoyal]
-* Fix broken rendering of variables which are equal to false, see #345 [Florian Weingarten, fw42]
-* Remove ActionView template handler [Dylan Thacker-Smith, dylanahsmith]
 * Freeze lots of string literals for new Ruby 2.1 optimization, see #297 [Florian Weingarten, fw42]
 * Allow newlines in tags and variables, see #324 [Dylan Thacker-Smith, dylanahsmith]
 * Tag#parse is called after initialize, which now takes options instead of tokens as the 3rd argument. See #321 [Dylan Thacker-Smith, dylanahsmith]
@@ -32,13 +26,7 @@
 * Make map filter work on enumerable drops, see #233 [Florian Weingarten, fw42]
 * Improved whitespace stripping for blank blocks, related to #216 [Florian Weingarten, fw42]
 
-## 2.6.1 / 2014-01-10 / branch "2-6-stable"
-
-Security fix, cherry-picked from master (4e14a65):
-* Don't call to_sym when creating conditions for security reasons, see #273 [Bouke van der Bijl, bouk]
-* Prevent arbitrary method invocation on condition objects, see #274 [Dylan Thacker-Smith, dylanahsmith]
-
-## 2.6.0 / 2013-11-25
+## 2.6.0 / 2013-11-25 / branch "2.6-stable"
 
 IMPORTANT: Liquid 2.6 is going to be the last version of Liquid which maintains explicit Ruby 1.8 compatability.
 The following releases will only be tested against Ruby 1.9 and Ruby 2.0 and are likely to break on Ruby 1.8.
@@ -62,13 +50,7 @@ The following releases will only be tested against Ruby 1.9 and Ruby 2.0 and are
 * Better documentation for 'include' tag (closes #163) [Peter Schröder, phoet]
 * Use of BigDecimal on filters to have better precision (closes #155) [Arthur Nogueira Neves, arthurnn]
 
-## 2.5.5 / 2014-01-10 / branch "2-5-stable"
-
-Security fix, cherry-picked from master (4e14a65):
-* Don't call to_sym when creating conditions for security reasons, see #273 [Bouke van der Bijl, bouk]
-* Prevent arbitrary method invocation on condition objects, see #274 [Dylan Thacker-Smith, dylanahsmith]
-
-## 2.5.4 / 2013-11-11
+## 2.5.4 / 2013-11-11 / branch "2.5-stable"
 
 * Fix "can't convert Fixnum into String" for "replace", see #173, [wǒ_is神仙, jsw0528]
 

README.md

@@ -1,6 +1,4 @@
 [![Build Status](https://secure.travis-ci.org/Shopify/liquid.png?branch=master)](http://travis-ci.org/Shopify/liquid)
-[![Inline docs](http://inch-ci.org/github/Shopify/liquid.png)](http://inch-ci.org/github/Shopify/liquid)
-
 # Liquid template engine
 
 * [Contributing guidelines](CONTRIBUTING.md)

Rakefile

@@ -8,7 +8,7 @@ task :default => 'test'
 desc 'run test suite with default parser'
 Rake::TestTask.new(:base_test) do |t|
   t.libs << '.' << 'lib' << 'test'
-  t.test_files = FileList['test/{integration,unit}/**/*_test.rb']
+  t.test_files = FileList['test/liquid/**/*_test.rb']
   t.verbose = false
 end
 
@@ -75,3 +75,11 @@ desc "Run example"
 task :example do
   ruby "-w -d -Ilib example/server/server.rb"
 end
+
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
+  require 'rake/extensiontask'
+  Rake::ExtensionTask.new "liquid" do |ext|
+    ext.lib_dir = "lib/liquid"
+  end
+  Rake::Task[:test].prerequisites << :compile
+end

example/server/templates/index.liquid

@@ -3,4 +3,4 @@
 <p>It is {{date}}</p>
 
 
-<p>Check out the <a href="/products">Products</a> screen </p>
+<p>Check out the <a href="http://localhost:3000/products">Products</a> screen </p>

ext/liquid/extconf.rb

@@ -0,0 +1,4 @@
+require 'mkmf'
+$CFLAGS << ' -Wall -Werror'
+$warnflags.gsub!(/-Wdeclaration-after-statement/, "")
+create_makefile("liquid/liquid")

ext/liquid/liquid.c

@@ -0,0 +1,9 @@
+#include "liquid.h"
+
+VALUE mLiquid;
+
+void Init_liquid(void)
+{
+    mLiquid = rb_define_module("Liquid");
+    init_liquid_tokenizer();
+}

ext/liquid/liquid.h

@@ -0,0 +1,11 @@
+#ifndef LIQUID_H
+#define LIQUID_H
+
+#include <ruby.h>
+#include <stdbool.h>
+
+#include "tokenizer.h"
+
+extern VALUE mLiquid;
+
+#endif

ext/liquid/tokenizer.c

@@ -0,0 +1,137 @@
+#include "liquid.h"
+
+VALUE cLiquidTokenizer;
+
+static void tokenizer_mark(void *ptr) {
+    tokenizer_t *tokenizer = ptr;
+    rb_gc_mark(tokenizer->source);
+}
+
+static void tokenizer_free(void *ptr)
+{
+    tokenizer_t *tokenizer = ptr;
+    xfree(tokenizer);
+}
+
+static size_t tokenizer_memsize(const void *ptr)
+{
+    return ptr ? sizeof(tokenizer_t) : 0;
+}
+
+const rb_data_type_t tokenizer_data_type = {
+    "liquid_tokenizer",
+    {tokenizer_mark, tokenizer_free, tokenizer_memsize,},
+#ifdef RUBY_TYPED_FREE_IMMEDIATELY
+    NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY
+#endif
+};
+
+static VALUE tokenizer_allocate(VALUE klass)
+{
+    VALUE obj;
+    tokenizer_t *tokenizer;
+
+    obj = TypedData_Make_Struct(klass, tokenizer_t, &tokenizer_data_type, tokenizer);
+    tokenizer->source = Qnil;
+    return obj;
+}
+
+static VALUE tokenizer_initialize_method(VALUE self, VALUE source)
+{
+    tokenizer_t *tokenizer;
+
+    Check_Type(source, T_STRING);
+    Tokenizer_Get_Struct(self, tokenizer);
+    source = rb_str_dup_frozen(source);
+    tokenizer->source = source;
+    tokenizer->cursor = RSTRING_PTR(source);
+    tokenizer->length = RSTRING_LEN(source);
+    return Qnil;
+}
+
+void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
+{
+    if (tokenizer->length <= 0) {
+        memset(token, 0, sizeof(*token));
+        return;
+    }
+
+    const char *cursor = tokenizer->cursor;
+    const char *last = cursor + tokenizer->length - 1;
+
+    token->str = cursor;
+    token->type = TOKEN_STRING;
+
+    while (cursor < last) {
+        if (*cursor++ != '{')
+            continue;
+
+        char c = *cursor++;
+        if (c != '%' && c != '{')
+            continue;
+        if (cursor - tokenizer->cursor > 2) {
+            token->type = TOKEN_STRING;
+            cursor -= 2;
+            goto found;
+        }
+        token->type = TOKEN_INVALID;
+        if (c == '%') {
+            while (cursor < last) {
+                if (*cursor++ != '%')
+                    continue;
+                c = *cursor++;
+                while (c == '%' && cursor <= last)
+                    c = *cursor++;
+                if (c != '}')
+                    continue;
+                token->type = TOKEN_TAG;
+                goto found;
+            }
+            // unterminated tag
+            cursor = tokenizer->cursor + 2;
+            goto found;
+        } else {
+            while (cursor < last) {
+                if (*cursor++ != '}')
+                    continue;
+                if (*cursor++ != '}') {
+                    // variable incomplete end, used to end raw tags
+                    cursor--;
+                    goto found;
+                }
+                token->type = TOKEN_VARIABLE;
+                goto found;
+            }
+            // unterminated variable
+            cursor = tokenizer->cursor + 2;
+            goto found;
+        }
+    }
+    cursor = last + 1;
+found:
+    token->length = cursor - tokenizer->cursor;
+    tokenizer->cursor += token->length;
+    tokenizer->length -= token->length;
+}
+
+static VALUE tokenizer_next_method(VALUE self)
+{
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(self, tokenizer);
+
+    token_t token;
+    tokenizer_next(tokenizer, &token);
+    if (token.type == TOKEN_NONE)
+        return Qnil;
+
+    return rb_str_new(token.str, token.length);
+}
+
+void init_liquid_tokenizer()
+{
+    cLiquidTokenizer = rb_define_class_under(mLiquid, "Tokenizer", rb_cObject);
+    rb_define_alloc_func(cLiquidTokenizer, tokenizer_allocate);
+    rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 1);
+    rb_define_method(cLiquidTokenizer, "next", tokenizer_next_method, 0);
+    rb_define_alias(cLiquidTokenizer, "shift", "next");
+}

ext/liquid/tokenizer.h

@@ -0,0 +1,31 @@
+#ifndef LIQUID_TOKENIZER_H
+#define LIQUID_TOKENIZER_H
+
+enum token_type {
+    TOKEN_NONE,
+    TOKEN_INVALID,
+    TOKEN_STRING,
+    TOKEN_TAG,
+    TOKEN_VARIABLE
+};
+
+typedef struct token {
+    enum token_type type;
+    const char *str;
+    long length;
+} token_t;
+
+typedef struct tokenizer {
+    VALUE source;
+    const char *cursor;
+    long length;
+} tokenizer_t;
+
+extern VALUE cLiquidTokenizer;
+extern const rb_data_type_t tokenizer_data_type;
+#define Tokenizer_Get_Struct(obj, sval) TypedData_Get_Struct(obj, tokenizer_t, &tokenizer_data_type, sval)
+
+void init_liquid_tokenizer();
+void tokenizer_next(tokenizer_t *tokenizer, token_t *token);
+
+#endif

init.rb

@@ -0,0 +1,8 @@
+require 'liquid'
+require 'extras/liquid_view'
+
+if defined? ActionView::Template and ActionView::Template.respond_to? :register_template_handler
+  ActionView::Template
+else
+  ActionView::Base
+end.register_template_handler(:liquid, LiquidView)

lib/extras/liquid_view.rb

@@ -0,0 +1,51 @@
+# LiquidView is a action view extension class. You can register it with rails
+# and use liquid as an template system for .liquid files
+#
+# Example
+#
+#   ActionView::Base::register_template_handler :liquid, LiquidView
+class LiquidView
+  PROTECTED_ASSIGNS = %w( template_root response _session template_class action_name request_origin session template
+                          _response url _request _cookies variables_added _flash params _headers request cookies
+                          ignore_missing_templates flash _params logger before_filter_chain_aborted headers )
+  PROTECTED_INSTANCE_VARIABLES = %w( @_request @controller @_first_render @_memoized__pick_template @view_paths
+                                     @helpers @assigns_added @template @_render_stack @template_format @assigns )
+
+  def self.call(template)
+    "LiquidView.new(self).render(template, local_assigns)"
+  end
+
+  def initialize(view)
+    @view = view
+  end
+
+  def render(template, local_assigns = nil)
+    @view.controller.headers["Content-Type"] ||= 'text/html; charset=utf-8'
+
+    # Rails 2.2 Template has source, but not locals
+    if template.respond_to?(:source) && !template.respond_to?(:locals)
+      assigns = (@view.instance_variables - PROTECTED_INSTANCE_VARIABLES).inject({}) do |hash, ivar|
+                  hash[ivar[1..-1]] = @view.instance_variable_get(ivar)
+                  hash
+                end
+    else
+      assigns = @view.assigns.reject{ |k,v| PROTECTED_ASSIGNS.include?(k) }
+    end
+
+    source = template.respond_to?(:source) ? template.source : template
+    local_assigns = (template.respond_to?(:locals) ? template.locals : local_assigns) || {}
+
+    if content_for_layout = @view.instance_variable_get("@content_for_layout")
+      assigns['content_for_layout'] = content_for_layout
+    end
+    assigns.merge!(local_assigns.stringify_keys)
+
+    liquid = Liquid::Template.parse(source)
+    liquid.render(assigns, :filters => [@view.controller.master_helper_module], :registers => {:action_view => @view, :controller => @view.controller})
+  end
+
+  def compilable?
+    false
+  end
+
+end

lib/liquid.rb

@@ -30,17 +30,10 @@ module Liquid
   VariableSegment             = /[\w\-]/
   VariableStart               = /\{\{/
   VariableEnd                 = /\}\}/
-  VariableIncompleteEnd       = /\}\}?/
   QuotedString                = /"[^"]*"|'[^']*'/
   QuotedFragment              = /#{QuotedString}|(?:[^\s,\|'"]|#{QuotedString})+/o
   TagAttributes               = /(\w+)\s*\:\s*(#{QuotedFragment})/o
-  AnyStartingTag              = /\{\{|\{\%/
-  PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om
-  TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/om
   VariableParser              = /\[[^\]]+\]|#{VariableSegment}+\??/o
-
-  singleton_class.send(:attr_accessor, :cache_classes)
-  self.cache_classes = true
 end
 
 require "liquid/version"
@@ -67,3 +60,9 @@ require 'liquid/utils'
 # Load all the tags of the standard library
 #
 Dir[File.dirname(__FILE__) + '/liquid/tags/*.rb'].each { |f| require f }
+
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
+  require 'liquid/liquid'
+else
+  require 'liquid/tokenizer'
+end

lib/liquid/block.rb

@@ -1,12 +1,12 @@
 module Liquid
   class Block < Tag
+    IsTag             = /\A#{TagStart}/o
+    IsVariable        = /\A#{VariableStart}/o
     FullToken         = /\A#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om
     ContentOfVariable = /\A#{VariableStart}(.*)#{VariableEnd}\z/om
-    TAGSTART = "{%".freeze
-    VARSTART = "{{".freeze
 
     def blank?
-      @blank
+      @blank || false
     end
 
     def parse(tokens)
@@ -18,41 +18,41 @@ module Liquid
       @children = []
 
       while token = tokens.shift
-        unless token.empty?
-          case
-          when token.start_with?(TAGSTART)
-            if token =~ FullToken
+        case token
+        when IsTag
+          if token =~ FullToken
 
-              # if we found the proper block delimiter just end parsing here and let the outer block
-              # proceed
-              if block_delimiter == $1
-                end_tag
-                return
-              end
-
-              # fetch the tag from registered blocks
-              if tag = Template.tags[$1]
-                new_tag = tag.parse($1, $2, tokens, @options)
-                @blank &&= new_tag.blank?
-                @nodelist << new_tag
-                @children << new_tag
-              else
-                # this tag is not registered with the system
-                # pass it to the current block for special handling or error reporting
-                unknown_tag($1, $2, tokens)
-              end
-            else
-              raise SyntaxError.new(options[:locale].t("errors.syntax.tag_termination".freeze, :token => token, :tag_end => TagEnd.inspect))
+            # if we found the proper block delimiter just end parsing here and let the outer block
+            # proceed
+            if block_delimiter == $1
+              end_tag
+              return
+            end
+
+            # fetch the tag from registered blocks
+            if tag = Template.tags[$1]
+              new_tag = tag.parse($1, $2, tokens, @options)
+              @blank &&= new_tag.blank?
+              @nodelist << new_tag
+              @children << new_tag
+            else
+              # this tag is not registered with the system
+              # pass it to the current block for special handling or error reporting
+              unknown_tag($1, $2, tokens)
             end
-          when token.start_with?(VARSTART)
-            new_var = create_variable(token)
-            @nodelist << new_var
-            @children << new_var
-            @blank = false
           else
-            @nodelist << token
-            @blank &&= (token =~ /\A\s*\z/)
+            raise SyntaxError.new(options[:locale].t("errors.syntax.tag_termination".freeze, :token => token, :tag_end => TagEnd.inspect))
           end
+        when IsVariable
+          new_var = create_variable(token)
+          @nodelist << new_var
+          @children << new_var
+          @blank = false
+        when ''.freeze
+          # pass
+        else
+          @nodelist << token
+          @blank &&= (token =~ /\A\s*\z/)
         end
       end
 
@@ -91,12 +91,12 @@ module Liquid
       end
     end
 
-    def block_name
-      @tag_name
+    def block_delimiter
+      "end#{block_name}"
     end
 
-    def block_delimiter
-      @block_delimiter ||= "end#{block_name}"
+    def block_name
+      @tag_name
     end
 
     def create_variable(token)

lib/liquid/condition.rb

@@ -94,16 +94,12 @@ module Liquid
 
       left, right = context[left], context[right]
 
-      operation = self.class.operators[op] || raise(Liquid::ArgumentError.new("Unknown operator #{op}"))
+      operation = self.class.operators[op] || raise(ArgumentError.new("Unknown operator #{op}"))
 
       if operation.respond_to?(:call)
         operation.call(self, left, right)
       elsif left.respond_to?(operation) and right.respond_to?(operation)
-        begin
-          left.send(operation, right)
-        rescue ::ArgumentError => e
-          raise Liquid::ArgumentError.new(e.message)
-        end
+        left.send(operation, right)
       else
         nil
       end

lib/liquid/context.rb

@@ -14,25 +14,18 @@ module Liquid
   #   context['bob']  #=> nil  class Context
   class Context
     attr_reader :scopes, :errors, :registers, :environments, :resource_limits
-    attr_accessor :exception_handler
-
-    SQUARE_BRACKETED = /\A\[(.*)\]\z/m
 
     def initialize(environments = {}, outer_scope = {}, registers = {}, rethrow_errors = false, resource_limits = {})
       @environments    = [environments].flatten
       @scopes          = [(outer_scope || {})]
       @registers       = registers
       @errors          = []
+      @rethrow_errors  = rethrow_errors
       @resource_limits = (resource_limits || {}).merge!({ :render_score_current => 0, :assign_score_current => 0 })
       squash_instance_assigns_with_environments
 
-      if rethrow_errors
-        self.exception_handler = ->(e) { true }
-      end
-
       @interrupts = []
       @filters = []
-      @parsed_variables = Hash.new{ |cache, markup| cache[markup] = variable_parse(markup) } 
     end
 
     def increment_used_resources(key, obj)
@@ -78,7 +71,7 @@ module Liquid
 
     # are there any not handled interrupts?
     def has_interrupt?
-      !@interrupts.empty?
+      @interrupts.any?
     end
 
     # push an interrupt to the stack. this interrupt is considered not handled.
@@ -93,8 +86,7 @@ module Liquid
 
     def handle_error(e)
       errors.push(e)
-
-      raise if exception_handler && exception_handler.call(e)
+      raise if @rethrow_errors
 
       case e
       when SyntaxError
@@ -197,18 +189,12 @@ module Liquid
 
       # Fetches an object starting at the local scope and then moving up the hierachy
       def find_variable(key)
-
-        # This was changed from find() to find_index() because this is a very hot
-        # path and find_index() is optimized in MRI to reduce object allocation
-        index = @scopes.find_index { |s| s.has_key?(key) }
-        scope = @scopes[index] if index
-
+        scope = @scopes.find { |s| s.has_key?(key) }
         variable = nil
 
         if scope.nil?
           @environments.each do |e|
-            variable = lookup_and_evaluate(e, key)
-            unless variable.nil?
+            if variable = lookup_and_evaluate(e, key)
               scope = e
               break
             end
@@ -224,16 +210,6 @@ module Liquid
         return variable
       end
 
-      def variable_parse(markup)
-        parts = markup.scan(VariableParser)
-        needs_resolution = false
-        if parts.first =~ SQUARE_BRACKETED 
-          needs_resolution = true
-          parts[0] = $1
-        end
-        {:first => parts.shift, :needs_resolution => needs_resolution, :rest => parts} 
-      end
-
       # Resolves namespaced queries gracefully.
       #
       # Example
@@ -241,17 +217,19 @@ module Liquid
       #  assert_equal 'tobi', @context['hash.name']
       #  assert_equal 'tobi', @context['hash["name"]']
       def variable(markup)
-        parts = @parsed_variables[markup]
+        parts = markup.scan(VariableParser)
+        square_bracketed = /\A\[(.*)\]\z/m
 
-        first_part = parts[:first] 
-        if parts[:needs_resolution]
-          first_part = resolve(parts[:first])
+        first_part = parts.shift
+
+        if first_part =~ square_bracketed
+          first_part = resolve($1)
         end
 
         if object = find_variable(first_part)
 
-          parts[:rest].each do |part|
-            part = resolve($1) if part_resolved = (part =~ SQUARE_BRACKETED)
+          parts.each do |part|
+            part = resolve($1) if part_resolved = (part =~ square_bracketed)
 
             # If object is a hash- or array-like object we look for the
             # presence of the key and if its available we return it
@@ -303,4 +281,5 @@ module Liquid
         end
       end # squash_instance_assigns_with_environments
   end # Context
+
 end # Liquid

lib/liquid/standardfilters.rb

@@ -92,16 +92,16 @@ module Liquid
 
     # Join elements of the array with certain character between them
     def join(input, glue = ' '.freeze)
-      InputIterator.new(input).join(glue)
+      [input].flatten.join(glue)
     end
 
     # Sort elements of the array
     # provide optional property with which to sort an array of hashes or drops
     def sort(input, property = nil)
-      ary = InputIterator.new(input)
+      ary = flatten_if_necessary(input)
       if property.nil?
         ary.sort
-      elsif ary.first.respond_to?('[]'.freeze) && !ary.first[property].nil?
+      elsif ary.first.respond_to?('[]'.freeze) and !ary.first[property].nil?
         ary.sort {|a,b| a[property] <=> b[property] }
       elsif ary.first.respond_to?(property)
         ary.sort {|a,b| a.send(property) <=> b.send(property) }
@@ -110,13 +110,13 @@ module Liquid
 
     # Reverse the elements of an array
     def reverse(input)
-      ary = InputIterator.new(input)
+      ary = [input].flatten
       ary.reverse
     end
 
     # map/collect on a given property
     def map(input, property)
-      InputIterator.new(input).map do |e|
+      flatten_if_necessary(input).map do |e|
         e = e.call if e.is_a?(Proc)
 
         if property == "to_liquid".freeze
@@ -162,7 +162,7 @@ module Liquid
       input.to_s.gsub(/\n/, "<br />\n".freeze)
     end
 
-    # Reformat a date using Ruby's core Time#strftime( string ) -> string
+    # Reformat a date
     #
     #   %a - The abbreviated weekday name (``Sun'')
     #   %A - The  full  weekday  name (``Sunday'')
@@ -176,7 +176,6 @@ module Liquid
     #   %m - Month of the year (01..12)
     #   %M - Minute of the hour (00..59)
     #   %p - Meridian indicator (``AM''  or  ``PM'')
-    #   %s - Number of seconds since 1970-01-01 00:00:00 UTC.
     #   %S - Second of the minute (00..60)
     #   %U - Week  number  of the current year,
     #           starting with the first Sunday as the first
@@ -191,14 +190,34 @@ module Liquid
     #   %Y - Year with century
     #   %Z - Time zone name
     #   %% - Literal ``%'' character
-    #
-    #   See also: http://www.ruby-doc.org/core/Time.html#method-i-strftime
     def date(input, format)
-      return input if format.to_s.empty?
 
-      return input unless date = to_date(input)
+      if format.to_s.empty?
+        return input.to_s
+      end
 
-      date.strftime(format.to_s)
+      if ((input.is_a?(String) && !/\A\d+\z/.match(input.to_s).nil?) || input.is_a?(Integer)) && input.to_i > 0
+        input = Time.at(input.to_i)
+      end
+
+      date = if input.is_a?(String)
+        case input.downcase
+        when 'now'.freeze, 'today'.freeze
+          Time.now
+        else
+          Time.parse(input)
+        end
+      else
+        input
+      end
+
+      if date.respond_to?(:strftime)
+        date.strftime(format.to_s)
+      else
+        input
+      end
+    rescue
+      input
     end
 
     # Get the first element of the passed in array
@@ -243,21 +262,6 @@ module Liquid
       apply_operation(input, operand, :%)
     end
 
-    def round(input, n = 0)
-      result = to_number(input).round(to_number(n))
-      result = result.to_f if result.is_a?(BigDecimal)
-      result = result.to_i if n == 0
-      result
-    end
-
-    def ceil(input)
-      to_number(input).ceil.to_i
-    end
-
-    def floor(input)
-      to_number(input).floor.to_i
-    end
-
     def default(input, default_value = "".freeze)
       is_blank = input.respond_to?(:empty?) ? input.empty? : !input
       is_blank ? default_value : input
@@ -265,6 +269,17 @@ module Liquid
 
     private
 
+    def flatten_if_necessary(input)
+      ary = if input.is_a?(Array)
+        input.flatten
+      elsif input.is_a?(Enumerable) && !input.is_a?(Hash)
+        input
+      else
+        [input].flatten
+      end
+      ary.map{ |e| e.respond_to?(:to_liquid) ? e.to_liquid : e }
+    end
+
     def to_number(obj)
       case obj
       when Float
@@ -278,57 +293,10 @@ module Liquid
       end
     end
 
-    def to_date(obj)
-      return obj if obj.respond_to?(:strftime)
-
-      case obj
-      when 'now'.freeze, 'today'.freeze
-        Time.now
-      when /\A\d+\z/, Integer
-        Time.at(obj.to_i)
-      when String
-        Time.parse(obj)
-      else
-        nil
-      end
-    rescue ArgumentError
-      nil
-    end
-
     def apply_operation(input, operand, operation)
       result = to_number(input).send(operation, to_number(operand))
       result.is_a?(BigDecimal) ? result.to_f : result
     end
-
-    class InputIterator
-      include Enumerable
-
-      def initialize(input)
-        @input = if input.is_a?(Array)
-          input.flatten
-        elsif input.is_a?(Hash)
-          [input]
-        elsif input.is_a?(Enumerable)
-          input
-        else
-          Array(input)
-        end
-      end
-
-      def join(glue)
-        to_a.join(glue)
-      end
-
-      def reverse
-        reverse_each.to_a
-      end
-
-      def each
-        @input.each do |e|
-          yield(e.respond_to?(:to_liquid) ? e.to_liquid : e)
-        end
-      end
-    end
   end
 
   Template.register_filter(StandardFilters)

lib/liquid/tag.rb

@@ -31,7 +31,7 @@ module Liquid
     end
 
     def blank?
-      false
+      @blank || false
     end
 
     def parse_with_selected_parser(markup)
@@ -50,12 +50,11 @@ module Liquid
     end
 
     private
-
     def strict_parse_with_error_context(markup)
       strict_parse(markup)
     rescue SyntaxError => e
       e.message << " in \"#{markup.strip}\"" 
       raise e
     end
-  end
-end
+  end # Tag
+end # Liquid

lib/liquid/tags/capture.rb

@@ -1,4 +1,5 @@
 module Liquid
+
   # Capture stores the result of a block into a variable without rendering it inplace.
   #
   #   {% capture heading %}

lib/liquid/tags/cycle.rb

@@ -43,8 +43,11 @@ module Liquid
       end
     end
 
-    private
+    def blank?
+      false
+    end
 
+    private
     def variables_from_string(markup)
       markup.split(',').collect do |var|
         var =~ /\s*(#{QuotedFragment})\s*/o

lib/liquid/tags/include.rb

@@ -38,6 +38,10 @@ module Liquid
     def parse(tokens)
     end
 
+    def blank?
+      false
+    end
+
     def render(context)
       partial = load_cached_partial(context)
       variable = context[@variable_name || @template_name[1..-2]]

lib/liquid/tags/increment.rb

@@ -25,6 +25,10 @@ module Liquid
       context.environments.first[@variable] = value + 1
       value.to_s
     end
+
+    def blank?
+      false
+    end
   end
 
   Template.register_tag('increment'.freeze, Increment)

lib/liquid/template.rb

@@ -21,43 +21,7 @@ module Liquid
     attr_accessor :root, :resource_limits
     @@file_system = BlankFileSystem.new
 
-    class TagRegistry
-      def initialize
-        @tags  = {}
-        @cache = {}
-      end
-
-      def [](tag_name)
-        return nil unless @tags.has_key?(tag_name)
-        return @cache[tag_name] if Liquid.cache_classes
-
-        lookup_class(@tags[tag_name]).tap { |o| @cache[tag_name] = o }
-      end
-
-      def []=(tag_name, klass)
-        @tags[tag_name]  = klass.name
-        @cache[tag_name] = klass
-      end
-
-      def delete(tag_name)
-        @tags.delete(tag_name)
-        @cache.delete(tag_name)
-      end
-
-      private
-
-      def lookup_class(name)
-        name.split("::").reject(&:empty?).reduce(Object) { |scope, const| scope.const_get(const) }
-      end
-    end
-
     class << self
-      # Sets how strict the parser should be.
-      # :lax acts like liquid 2.5 and silently ignores malformed tags in most cases.
-      # :warn is the default and will give deprecation warnings when invalid syntax is used.
-      # :strict will enforce correct syntax.
-      attr_writer :error_mode
-
       def file_system
         @@file_system
       end
@@ -71,7 +35,15 @@ module Liquid
       end
 
       def tags
-        @tags ||= TagRegistry.new
+        @tags ||= {}
+      end
+
+      # Sets how strict the parser should be.
+      # :lax acts like liquid 2.5 and silently ignores malformed tags in most cases.
+      # :warn is the default and will give deprecation warnings when invalid syntax is used.
+      # :strict will enforce correct syntax.
+      def error_mode=(mode)
+        @error_mode = mode
       end
 
       def error_mode
@@ -88,6 +60,7 @@ module Liquid
       def parse(source, options = {})
         template = Template.new
         template.parse(source, options)
+        template
       end
     end
 
@@ -141,13 +114,7 @@ module Liquid
 
       context = case args.first
       when Liquid::Context
-        c = args.shift
-
-        if @rethrow_errors
-          c.exception_handler = ->(e) { true }
-        end
-
-        c
+        args.shift
       when Liquid::Drop
         drop = args.shift
         drop.context = Context.new([drop, assigns], instance_assigns, registers, @rethrow_errors, @resource_limits)
@@ -171,9 +138,6 @@ module Liquid
           context.add_filters(options[:filters])
         end
 
-        if options[:exception_handler]
-          context.exception_handler = options[:exception_handler]
-        end
       when Module
         context.add_filters(args.pop)
       when Array
@@ -193,22 +157,14 @@ module Liquid
     end
 
     def render!(*args)
-      @rethrow_errors = true
-      render(*args)
+      @rethrow_errors = true; render(*args)
     end
 
     private
 
-    # Uses the <tt>Liquid::TemplateParser</tt> regexp to tokenize the passed source
     def tokenize(source)
       source = source.source if source.respond_to?(:source)
-      return [] if source.to_s.empty?
-      tokens = source.split(TemplateParser)
-
-      # removes the rogue empty element at the beginning of the array
-      tokens.shift if tokens[0] and tokens[0].empty?
-
-      tokens
+      Tokenizer.new(source.to_s)
     end
 
   end

lib/liquid/tokenizer.rb

@@ -0,0 +1,20 @@
+module Liquid
+  class Tokenizer
+    VariableIncompleteEnd = /\}\}?/
+    AnyStartingTag        = /\{\{|\{\%/
+    PartialTemplateParser = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om
+    TemplateParser        = /(#{PartialTemplateParser}|#{AnyStartingTag})/om
+
+    def initialize(source)
+      @tokens = source.split(TemplateParser)
+
+      # removes the rogue empty element at the beginning of the array
+      @tokens.shift if @tokens[0] && @tokens[0].empty?
+    end
+
+    def next
+      @tokens.shift
+    end
+    alias_method :shift, :next
+  end
+end

lib/liquid/variable.rb

@@ -36,13 +36,13 @@ module Liquid
 
     def lax_parse(markup)
       @filters = []
-      if markup =~ /\s*(#{QuotedFragment})(.*)/om
-        @name = Regexp.last_match(1)
-        if Regexp.last_match(2) =~ /#{FilterSeparator}\s*(.*)/om
+      if match = markup.match(/\s*(#{QuotedFragment})(.*)/om)
+        @name = match[1]
+        if match[2].match(/#{FilterSeparator}\s*(.*)/om)
           filters = Regexp.last_match(1).scan(FilterParser)
           filters.each do |f|
-            if f =~ /\s*(\w+)/
-              filtername = Regexp.last_match(1)
+            if matches = f.match(/\s*(\w+)/)
+              filtername = matches[1]
               filterargs = f.scan(/(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o).flatten
               @filters << [filtername, filterargs]
             end

lib/liquid/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Liquid
-  VERSION = "3.0.0.rc1"
+  VERSION = "3.0.0".freeze
 end

liquid.gemspec

@@ -18,11 +18,17 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = ">= 1.3.7"
 
   s.test_files  = Dir.glob("{test}/**/*")
-  s.files       = Dir.glob("{lib}/**/*") + %w(MIT-LICENSE README.md)
+  s.files       = Dir.glob("{lib,ext}/**/*") + %w(MIT-LICENSE README.md)
 
   s.extra_rdoc_files  = ["History.md", "README.md"]
 
   s.require_path = "lib"
 
   s.add_development_dependency 'rake'
+  s.add_development_dependency 'activesupport'
+  if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
+    s.extensions  = ['ext/liquid/extconf.rb']
+    s.add_development_dependency 'rake-compiler'
+    s.add_development_dependency 'stackprof' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0")
+  end
 end

performance/benchmark.rb

@@ -1,3 +1,4 @@
+require 'rubygems'
 require 'benchmark'
 require File.dirname(__FILE__) + '/theme_runner'
 

performance/profile.rb

@@ -4,14 +4,10 @@ require File.dirname(__FILE__) + '/theme_runner'
 Liquid::Template.error_mode = ARGV.first.to_sym if ARGV.first
 profiler = ThemeRunner.new
 profiler.run
-
-[:cpu, :object].each do |profile_type|
-  puts "Profiling in #{profile_type.to_s} mode..."
-  results = StackProf.run(mode: profile_type) do
-    100.times do
-      profiler.run
-    end
+results = StackProf.run(mode: :cpu) do
+  100.times do
+    profiler.run
   end
-  StackProf::Report.new(results).print_text(false, 20)
-  File.write(ENV['FILENAME'] + "." + profile_type.to_s, Marshal.dump(results)) if ENV['FILENAME']
 end
+StackProf::Report.new(results).print_text(false, 20)
+File.write(ENV['FILENAME'], Marshal.dump(results)) if ENV['FILENAME']

performance/shopify/database.rb

@@ -1,6 +1,6 @@
 require 'yaml'
-
 module Database
+
   # Load the standard vision toolkit database and re-arrage it to be simply exportable
   # to liquid as assigns. All this is based on Shopify
   def self.tables

performance/shopify/json_filter.rb

@@ -1,9 +1,7 @@
-require 'json'
-
 module JsonFilter
 
   def json(object)
-    JSON.dump(object.reject {|k,v| k == "collections" })
+    object.reject {|k,v| k == "collections" }.to_json
   end
 
 end

performance/theme_runner.rb

@@ -6,6 +6,11 @@
 # Shopify which is likely the biggest user of liquid in the world which something to the tune of several
 # million Template#render calls a day.
 
+require 'rubygems'
+require 'active_support'
+require 'active_support/json'
+require 'yaml'
+require 'digest/md5'
 require File.dirname(__FILE__) + '/shopify/liquid'
 require File.dirname(__FILE__) + '/shopify/database.rb'
 
@@ -76,3 +81,6 @@ class ThemeRunner
     end
   end
 end
+
+
+

test/integration/context_test.rb

@@ -1,33 +0,0 @@
-require 'test_helper'
-
-class ContextTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_override_global_filter
-    global = Module.new do
-      def notice(output)
-        "Global #{output}"
-      end
-    end
-
-    local = Module.new do
-      def notice(output)
-        "Local #{output}"
-      end
-    end
-
-    Template.register_filter(global)
-    assert_equal 'Global test', Template.parse("{{'test' | notice }}").render!
-    assert_equal 'Local test', Template.parse("{{'test' | notice }}").render!({}, :filters => [local])
-  end
-
-  def test_has_key_will_not_add_an_error_for_missing_keys
-    Template.error_mode = :strict
-
-    context = Context.new
-
-    context.has_key?('unknown')
-
-    assert_empty context.errors
-  end
-end

test/integration/variable_test.rb

@@ -1,72 +0,0 @@
-require 'test_helper'
-
-class VariableTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_simple_variable
-    template = Template.parse(%|{{test}}|)
-    assert_equal 'worked', template.render!('test' => 'worked')
-    assert_equal 'worked wonderfully', template.render!('test' => 'worked wonderfully')
-  end
-
-  def test_simple_with_whitespaces
-    template = Template.parse(%|  {{ test }}  |)
-    assert_equal '  worked  ', template.render!('test' => 'worked')
-    assert_equal '  worked wonderfully  ', template.render!('test' => 'worked wonderfully')
-  end
-
-  def test_ignore_unknown
-    template = Template.parse(%|{{ test }}|)
-    assert_equal '', template.render!
-  end
-
-  def test_hash_scoping
-    template = Template.parse(%|{{ test.test }}|)
-    assert_equal 'worked', template.render!('test' => {'test' => 'worked'})
-  end
-
-  def test_false_renders_as_false
-    assert_equal 'false', Template.parse("{{ foo }}").render!('foo' => false)
-  end
-
-  def test_preset_assigns
-    template = Template.parse(%|{{ test }}|)
-    template.assigns['test'] = 'worked'
-    assert_equal 'worked', template.render!
-  end
-
-  def test_reuse_parsed_template
-    template = Template.parse(%|{{ greeting }} {{ name }}|)
-    template.assigns['greeting'] = 'Goodbye'
-    assert_equal 'Hello Tobi', template.render!('greeting' => 'Hello', 'name' => 'Tobi')
-    assert_equal 'Hello ', template.render!('greeting' => 'Hello', 'unknown' => 'Tobi')
-    assert_equal 'Hello Brian', template.render!('greeting' => 'Hello', 'name' => 'Brian')
-    assert_equal 'Goodbye Brian', template.render!('name' => 'Brian')
-    assert_equal({'greeting'=>'Goodbye'}, template.assigns)
-  end
-
-  def test_assigns_not_polluted_from_template
-    template = Template.parse(%|{{ test }}{% assign test = 'bar' %}{{ test }}|)
-    template.assigns['test'] = 'baz'
-    assert_equal 'bazbar', template.render!
-    assert_equal 'bazbar', template.render!
-    assert_equal 'foobar', template.render!('test' => 'foo')
-    assert_equal 'bazbar', template.render!
-  end
-
-  def test_hash_with_default_proc
-    template = Template.parse(%|Hello {{ test }}|)
-    assigns = Hash.new { |h,k| raise "Unknown variable '#{k}'" }
-    assigns['test'] = 'Tobi'
-    assert_equal 'Hello Tobi', template.render!(assigns)
-    assigns.delete('test')
-    e = assert_raises(RuntimeError) {
-      template.render!(assigns)
-    }
-    assert_equal "Unknown variable 'test'", e.message
-  end
-
-  def test_multiline_variable
-    assert_equal 'worked', Template.parse("{{\ntest\n}}").render!('test' => 'worked')
-  end
-end

test/liquid/block_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class BlockUnitTest < Test::Unit::TestCase
+class BlockTest < Test::Unit::TestCase
   include Liquid
 
   def test_blankspace

test/liquid/condition_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class ConditionUnitTest < Test::Unit::TestCase
+class ConditionTest < Test::Unit::TestCase
   include Liquid
 
   def test_basic_condition
@@ -49,17 +49,6 @@ class ConditionUnitTest < Test::Unit::TestCase
     assert_evalutes_false "'bob'", 'contains', "'---'"
   end
 
-  def test_invalid_comparation_operator
-    assert_evaluates_argument_error "1", '~~', '0'
-  end
-
-  def test_comparation_of_int_and_str
-    assert_evaluates_argument_error "'1'", '>', '0'
-    assert_evaluates_argument_error "'1'", '<', '0'
-    assert_evaluates_argument_error "'1'", '>=', '0'
-    assert_evaluates_argument_error "'1'", '<=', '0'
-  end
-
   def test_contains_works_on_arrays
     @context = Liquid::Context.new
     @context['array'] = [1,2,3,4,5]
@@ -135,11 +124,4 @@ class ConditionUnitTest < Test::Unit::TestCase
       assert !Condition.new(left, op, right).evaluate(@context || Liquid::Context.new),
              "Evaluated true: #{left} #{op} #{right}"
     end
-
-    def assert_evaluates_argument_error(left, op, right)
-      assert_raises(Liquid::ArgumentError) do
-        Condition.new(left, op, right).evaluate(@context || Liquid::Context.new)
-      end
-    end
-
 end # ConditionTest

test/liquid/context_test.rb

@@ -63,17 +63,13 @@ class ArrayLike
   end
 end
 
-class ContextUnitTest < Test::Unit::TestCase
+class ContextTest < Test::Unit::TestCase
   include Liquid
 
   def setup
     @context = Liquid::Context.new
   end
 
-  def teardown
-    Spy.teardown
-  end
-
   def test_variables
     @context['string'] = 'string'
     assert_equal 'string', @context['string']
@@ -166,6 +162,24 @@ class ContextUnitTest < Test::Unit::TestCase
 
   end
 
+  def test_override_global_filter
+    global = Module.new do
+      def notice(output)
+        "Global #{output}"
+      end
+    end
+
+    local = Module.new do
+      def notice(output)
+        "Local #{output}"
+      end
+    end
+
+    Template.register_filter(global)
+    assert_equal 'Global test', Template.parse("{{'test' | notice }}").render!
+    assert_equal 'Local test', Template.parse("{{'test' | notice }}").render!({}, :filters => [local])
+  end
+
   def test_only_intended_filters_make_it_there
 
     filter = Module.new do
@@ -461,26 +475,4 @@ class ContextUnitTest < Test::Unit::TestCase
     assert_kind_of CategoryDrop, @context['category']
     assert_equal @context, @context['category'].context
   end
-
-  def test_use_empty_instead_of_any_in_interrupt_handling_to_avoid_lots_of_unnecessary_object_allocations
-    mock_any = Spy.on_instance_method(Array, :any?)
-    mock_empty = Spy.on_instance_method(Array, :empty?)
-    mock_has_interrupt = Spy.on(@context, :has_interrupt?).and_call_through
-
-    @context.has_interrupt?
-
-    refute mock_any.has_been_called?
-    assert mock_empty.has_been_called?
-  end 
-
-  def test_variable_lookup_caches_markup
-    mock_scan = Spy.on_instance_method(String, :scan).and_return(["string"])
-
-    @context['string'] = 'string'
-    @context['string']
-    @context['string']
-
-    assert_equal 1, mock_scan.calls.size
-  end
-
 end # ContextTest

test/liquid/file_system_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class FileSystemUnitTest < Test::Unit::TestCase
+class FileSystemTest < Test::Unit::TestCase
   include Liquid
 
   def test_default

test/liquid/filter_test.rb

@@ -67,12 +67,12 @@ class FiltersTest < Test::Unit::TestCase
     @context['value'] = 3
     @context['numbers'] = [2,1,4,3]
     @context['words'] = ['expected', 'as', 'alphabetic']
-    @context['arrays'] = ['flower', 'are']
+    @context['arrays'] = [['flattened'], ['are']]
 
     assert_equal [1,2,3,4], Variable.new("numbers | sort").render(@context)
     assert_equal ['alphabetic', 'as', 'expected'], Variable.new("words | sort").render(@context)
     assert_equal [3], Variable.new("value | sort").render(@context)
-    assert_equal ['are', 'flower'], Variable.new("arrays | sort").render(@context)
+    assert_equal ['are', 'flattened'], Variable.new("arrays | sort").render(@context)
   end
 
   def test_strip_html

test/liquid/i18n_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class I18nUnitTest < Test::Unit::TestCase
+class I18nTest < Test::Unit::TestCase
   include Liquid
 
   def setup

test/liquid/lexer_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class LexerUnitTest < Test::Unit::TestCase
+class LexerTest < Test::Unit::TestCase
   include Liquid
 
   def test_strings

test/liquid/module_ex_test.rb

@@ -36,7 +36,7 @@ class TestClassC::LiquidDropClass
   end
 end
 
-class ModuleExUnitTest < Test::Unit::TestCase
+class ModuleExTest < Test::Unit::TestCase
   include Liquid
 
   def setup

test/liquid/parser_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class ParserUnitTest < Test::Unit::TestCase
+class ParserTest < Test::Unit::TestCase
   include Liquid
 
   def test_consume

test/liquid/parsing_quirks_test.rb

@@ -3,9 +3,12 @@ require 'test_helper'
 class ParsingQuirksTest < Test::Unit::TestCase
   include Liquid
 
-  def test_parsing_css
-    text = " div { font-weight: bold; } "
-    assert_equal text, Template.parse(text).render!
+  def test_error_with_css
+    text = %| div { font-weight: bold; } |
+    template = Template.parse(text)
+
+    assert_equal text, template.render!
+    assert_equal [String], template.root.nodelist.collect {|i| i.class}
   end
 
   def test_raise_on_single_close_bracet
@@ -84,11 +87,4 @@ class ParsingQuirksTest < Test::Unit::TestCase
       assert_template_result('',"{% if #{markup} %} YES {% endif %}")
     end
   end
-
-  def test_raise_on_invalid_tag_delimiter
-    assert_raise(Liquid::SyntaxError) do
-      Template.new.parse('{% end %}')
-    end
-  end
-
 end # ParsingQuirksTest

test/liquid/regexp_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class RegexpUnitTest < Test::Unit::TestCase
+class RegexpTest < Test::Unit::TestCase
   include Liquid
 
   def test_empty

test/liquid/security_test.rb

@@ -54,10 +54,10 @@ class SecurityTest < Test::Unit::TestCase
   def test_does_not_add_drop_methods_to_symbol_table
     current_symbols = Symbol.all_symbols
 
-    assigns = { 'drop' => Drop.new }
-    assert_equal "", Template.parse("{{ drop.custom_method_1 }}", assigns).render!
-    assert_equal "", Template.parse("{{ drop.custom_method_2 }}", assigns).render!
-    assert_equal "", Template.parse("{{ drop.custom_method_3 }}", assigns).render!
+    drop = Drop.new
+    drop.invoke_drop("custom_method_1")
+    drop.invoke_drop("custom_method_2")
+    drop.invoke_drop("custom_method_3")
 
     assert_equal [], (Symbol.all_symbols - current_symbols)
   end

test/liquid/standard_filter_test.rb

@@ -7,8 +7,6 @@ class Filters
 end
 
 class TestThing
-  attr_reader :foo
-
   def initialize
     @foo = 0
   end
@@ -117,25 +115,10 @@ class StandardFiltersTest < Test::Unit::TestCase
     assert_equal [{"a" => 1}, {"a" => 2}, {"a" => 3}, {"a" => 4}], @filters.sort([{"a" => 4}, {"a" => 3}, {"a" => 1}, {"a" => 2}], "a")
   end
 
-  def test_legacy_sort_hash
-    assert_equal [{a:1, b:2}], @filters.sort({a:1, b:2})
-  end
-
-  def test_numerical_vs_lexicographical_sort
-    assert_equal [2, 10], @filters.sort([10, 2])
-    assert_equal [{"a" => 2}, {"a" => 10}], @filters.sort([{"a" => 10}, {"a" => 2}], "a")
-    assert_equal ["10", "2"], @filters.sort(["10", "2"])
-    assert_equal [{"a" => "10"}, {"a" => "2"}], @filters.sort([{"a" => "10"}, {"a" => "2"}], "a")
-  end
-
   def test_reverse
     assert_equal [4,3,2,1], @filters.reverse([1,2,3,4])
   end
 
-  def test_legacy_reverse_hash
-    assert_equal [{a:1, b:2}], @filters.reverse(a:1, b:2)
-  end
-
   def test_map
     assert_equal [1,2,3,4], @filters.map([{"a" => 1}, {"a" => 2}, {"a" => 3}, {"a" => 4}], 'a')
     assert_template_result 'abc', "{{ ary | map:'foo' | map:'bar' }}",
@@ -157,16 +140,9 @@ class StandardFiltersTest < Test::Unit::TestCase
       "thing" => { "foo" => [ { "bar" => 42 }, { "bar" => 17 } ] }
   end
 
-  def test_legacy_map_on_hashes_with_dynamic_key
-    template = "{% assign key = 'foo' %}{{ thing | map: key | map: 'bar' }}"
-    hash = { "foo" => { "bar" => 42 } }
-    assert_template_result "42", template, "thing" => hash
-  end
-
   def test_sort_calls_to_liquid
     t = TestThing.new
-    Liquid::Template.parse('{{ foo | sort: "whatever" }}').render("foo" => [t])
-    assert t.foo > 0
+    assert_template_result "woot: 1", '{{ foo | sort: "whatever" }}', "foo" => [t]
   end
 
   def test_map_over_proc
@@ -210,6 +186,7 @@ class StandardFiltersTest < Test::Unit::TestCase
     assert_equal "07/05/2006", @filters.date("1152098955", "%m/%d/%Y")
   end
 
+
   def test_first_last
     assert_equal 1, @filters.first([1,2,3])
     assert_equal 3, @filters.last([1,2,3])
@@ -290,22 +267,6 @@ class StandardFiltersTest < Test::Unit::TestCase
     assert_template_result "1", "{{ 3 | modulo:2 }}"
   end
 
-  def test_round
-    assert_template_result "5", "{{ input | round }}", 'input' => 4.6
-    assert_template_result "4", "{{ '4.3' | round }}"
-    assert_template_result "4.56", "{{ input | round: 2 }}", 'input' => 4.5612
-  end
-
-  def test_ceil
-    assert_template_result "5", "{{ input | ceil }}", 'input' => 4.6
-    assert_template_result "5", "{{ '4.3' | ceil }}"
-  end
-
-  def test_floor
-    assert_template_result "4", "{{ input | floor }}", 'input' => 4.6
-    assert_template_result "4", "{{ '4.3' | floor }}"
-  end
-
   def test_append
     assigns = {'a' => 'bc', 'b' => 'd' }
     assert_template_result('bcd',"{{ a | append: 'd'}}",assigns)

test/liquid/strainer_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class StrainerUnitTest < Test::Unit::TestCase
+class StrainerTest < Test::Unit::TestCase
   include Liquid
 
   module AccessScopeFilters

test/liquid/tags/case_tag_test.rb

@@ -1,10 +1,10 @@
 require 'test_helper'
 
-class CaseTagUnitTest < Test::Unit::TestCase
+class CaseTagTest < Test::Unit::TestCase
   include Liquid
 
   def test_case_nodelist
     template = Liquid::Template.parse('{% case var %}{% when true %}WHEN{% else %}ELSE{% endcase %}')
     assert_equal ['WHEN', 'ELSE'], template.root.nodelist[0].nodelist
   end
-end
+end # CaseTest

test/liquid/tags/for_tag_test.rb

@@ -1,11 +1,5 @@
 require 'test_helper'
 
-class ThingWithValue < Liquid::Drop
-  def value
-    3
-  end
-end
-
 class ForTagTest < Test::Unit::TestCase
   include Liquid
 
@@ -40,20 +34,6 @@ HERE
     assert_template_result(' 1  2  3 ','{%for item in (1..3) %} {{item}} {%endfor%}')
   end
 
-  def test_for_with_variable_range
-    assert_template_result(' 1  2  3 ','{%for item in (1..foobar) %} {{item}} {%endfor%}', "foobar" => 3)
-  end
-
-  def test_for_with_hash_value_range
-    foobar = { "value" => 3 }
-    assert_template_result(' 1  2  3 ','{%for item in (1..foobar.value) %} {{item}} {%endfor%}', "foobar" => foobar)
-  end
-
-  def test_for_with_drop_value_range
-    foobar = ThingWithValue.new
-    assert_template_result(' 1  2  3 ','{%for item in (1..foobar.value) %} {{item}} {%endfor%}', "foobar" => foobar)
-  end
-
   def test_for_with_variable
     assert_template_result(' 1  2  3 ','{%for item in array%} {{item}} {%endfor%}','array' => [1,2,3])
     assert_template_result('123','{%for item in array%}{{item}}{%endfor%}','array' => [1,2,3])
@@ -315,6 +295,16 @@ HERE
     assert_template_result(expected, template, assigns)
   end
 
+  def test_for_nodelist
+    template = Liquid::Template.parse('{% for item in items %}FOR{% endfor %}')
+    assert_equal ['FOR'], template.root.nodelist[0].nodelist
+  end
+
+  def test_for_else_nodelist
+    template = Liquid::Template.parse('{% for item in items %}FOR{% else %}ELSE{% endfor %}')
+    assert_equal ['FOR', 'ELSE'], template.root.nodelist[0].nodelist
+  end
+
   class LoaderDrop < Liquid::Drop
     attr_accessor :each_called, :load_slice_called
 

test/liquid/tags/if_else_tag_test.rb

@@ -158,9 +158,14 @@ class IfElseTagTest < Test::Unit::TestCase
                            %({% if 'gnomeslab-and-or-liquid' contains 'gnomeslab-and-or-liquid' %}yes{% endif %}))
   end
 
+  def test_if_nodelist
+    template = Liquid::Template.parse('{% if true %}IF{% else %}ELSE{% endif %}')
+    assert_equal ['IF', 'ELSE'], template.root.nodelist[0].nodelist
+  end
+
   def test_operators_are_whitelisted
     assert_raise(SyntaxError) do
       assert_template_result('', %({% if 1 or throw or or 1 %}yes{% endif %}))
     end
   end
-end
+end # IfElseTest

test/liquid/tags/include_tag_test.rb

@@ -60,6 +60,10 @@ class CustomInclude < Liquid::Tag
   def parse(tokens)
   end
 
+  def blank?
+    false
+  end
+
   def render(context)
     @template_name[1..-2]
   end
@@ -201,12 +205,4 @@ class IncludeTagTest < Test::Unit::TestCase
       Liquid::Template.tags['include'] = original_tag
     end
   end
-
-  def test_does_not_add_error_in_strict_mode_for_missing_variable
-    Liquid::Template.file_system = TestFileSystem.new
-
-    a = Liquid::Template.parse(' {% include "nested_template" %}')
-    a.render!
-    assert_empty a.errors
-  end
 end # IncludeTagTest

test/liquid/tags/standard_tag_test.rb

@@ -3,6 +3,12 @@ require 'test_helper'
 class StandardTagTest < Test::Unit::TestCase
   include Liquid
 
+  def test_tag
+    tag = Tag.parse('tag', [], [], {})
+    assert_equal 'liquid::tag', tag.name
+    assert_equal '', tag.render(Context.new)
+  end
+
   def test_no_transform
     assert_template_result('this text should come out of the template without change...',
                            'this text should come out of the template without change...')

test/liquid/template_test.rb

@@ -22,12 +22,6 @@ class SomethingWithLength
   liquid_methods :length
 end
 
-class ErroneousDrop < Liquid::Drop
-  def bad_method
-    raise 'ruby error in drop'
-  end
-end
-
 class TemplateTest < Test::Unit::TestCase
   include Liquid
 
@@ -144,27 +138,17 @@ class TemplateTest < Test::Unit::TestCase
     assert_equal 'haha', t.parse("{{baz}}").render!(drop)
   end
 
-  def test_render_bang_force_rethrow_errors_on_passed_context
-    context = Context.new({'drop' => ErroneousDrop.new})
-    t = Template.new.parse('{{ drop.bad_method }}')
-
-    e = assert_raises RuntimeError do
-      t.render!(context)
-    end
-    assert_equal 'ruby error in drop', e.message
+  def test_sets_default_localization_in_document
+    t = Template.new
+    t.parse('')
+    assert_instance_of I18n, t.root.options[:locale]
   end
 
-  def test_exception_handler_doesnt_reraise_if_it_returns_false
-    exception = nil
-    Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_handler: ->(e) { exception = e; false })
-    assert exception.is_a?(ZeroDivisionError)
-  end
+  def test_sets_default_localization_in_context_with_quick_initialization
+    t = Template.new
+    t.parse('{{foo}}', :locale => I18n.new(fixture("en_locale.yml")))
 
-  def test_exception_handler_does_reraise_if_it_returns_true
-    exception = nil
-    assert_raises(ZeroDivisionError) do
-      Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_handler: ->(e) { exception = e; true })
-    end
-    assert exception.is_a?(ZeroDivisionError)
+    assert_instance_of I18n, t.root.options[:locale]
+    assert_equal fixture("en_locale.yml"), t.root.options[:locale].path
   end
 end

test/liquid/tokenizer_test.rb

@@ -24,6 +24,11 @@ class TokenizerTest < Test::Unit::TestCase
   private
 
   def tokenize(source)
-    Liquid::Template.new.send(:tokenize, source)
+    tokenizer = Liquid::Tokenizer.new(source)
+    tokens = []
+    while token = tokenizer.next
+      tokens << token
+    end
+    tokens
   end
 end

test/liquid/variable_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class VariableUnitTest < Test::Unit::TestCase
+class VariableTest < Test::Unit::TestCase
   include Liquid
 
   def test_variable
@@ -134,3 +134,71 @@ class VariableUnitTest < Test::Unit::TestCase
     end
   end
 end
+
+
+class VariableResolutionTest < Test::Unit::TestCase
+  include Liquid
+
+  def test_simple_variable
+    template = Template.parse(%|{{test}}|)
+    assert_equal 'worked', template.render!('test' => 'worked')
+    assert_equal 'worked wonderfully', template.render!('test' => 'worked wonderfully')
+  end
+
+  def test_simple_with_whitespaces
+    template = Template.parse(%|  {{ test }}  |)
+    assert_equal '  worked  ', template.render!('test' => 'worked')
+    assert_equal '  worked wonderfully  ', template.render!('test' => 'worked wonderfully')
+  end
+
+  def test_ignore_unknown
+    template = Template.parse(%|{{ test }}|)
+    assert_equal '', template.render!
+  end
+
+  def test_hash_scoping
+    template = Template.parse(%|{{ test.test }}|)
+    assert_equal 'worked', template.render!('test' => {'test' => 'worked'})
+  end
+
+  def test_preset_assigns
+    template = Template.parse(%|{{ test }}|)
+    template.assigns['test'] = 'worked'
+    assert_equal 'worked', template.render!
+  end
+
+  def test_reuse_parsed_template
+    template = Template.parse(%|{{ greeting }} {{ name }}|)
+    template.assigns['greeting'] = 'Goodbye'
+    assert_equal 'Hello Tobi', template.render!('greeting' => 'Hello', 'name' => 'Tobi')
+    assert_equal 'Hello ', template.render!('greeting' => 'Hello', 'unknown' => 'Tobi')
+    assert_equal 'Hello Brian', template.render!('greeting' => 'Hello', 'name' => 'Brian')
+    assert_equal 'Goodbye Brian', template.render!('name' => 'Brian')
+    assert_equal({'greeting'=>'Goodbye'}, template.assigns)
+  end
+
+  def test_assigns_not_polluted_from_template
+    template = Template.parse(%|{{ test }}{% assign test = 'bar' %}{{ test }}|)
+    template.assigns['test'] = 'baz'
+    assert_equal 'bazbar', template.render!
+    assert_equal 'bazbar', template.render!
+    assert_equal 'foobar', template.render!('test' => 'foo')
+    assert_equal 'bazbar', template.render!
+  end
+
+  def test_hash_with_default_proc
+    template = Template.parse(%|Hello {{ test }}|)
+    assigns = Hash.new { |h,k| raise "Unknown variable '#{k}'" }
+    assigns['test'] = 'Tobi'
+    assert_equal 'Hello Tobi', template.render!(assigns)
+    assigns.delete('test')
+    e = assert_raises(RuntimeError) {
+      template.render!(assigns)
+    }
+    assert_equal "Unknown variable 'test'", e.message
+  end
+
+  def test_multiline_variable
+    assert_equal 'worked', Template.parse("{{\ntest\n}}").render!('test' => 'worked')
+  end
+end # VariableTest

test/test_helper.rb

@@ -2,7 +2,6 @@
 
 require 'test/unit'
 require 'test/unit/assertions'
-require 'spy/integration'
 
 $:.unshift(File.join(File.expand_path(File.dirname(__FILE__)), '..', 'lib'))
 require 'liquid.rb'

test/unit/tag_unit_test.rb

@@ -1,11 +0,0 @@
-require 'test_helper'
-
-class TagUnitTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_tag
-    tag = Tag.parse('tag', [], [], {})
-    assert_equal 'liquid::tag', tag.name
-    assert_equal '', tag.render(Context.new)
-  end
-end

test/unit/tags/for_tag_unit_test.rb

@@ -1,13 +0,0 @@
-require 'test_helper'
-
-class ForTagUnitTest < Test::Unit::TestCase
-  def test_for_nodelist
-    template = Liquid::Template.parse('{% for item in items %}FOR{% endfor %}')
-    assert_equal ['FOR'], template.root.nodelist[0].nodelist
-  end
-
-  def test_for_else_nodelist
-    template = Liquid::Template.parse('{% for item in items %}FOR{% else %}ELSE{% endfor %}')
-    assert_equal ['FOR', 'ELSE'], template.root.nodelist[0].nodelist
-  end
-end

test/unit/tags/if_tag_unit_test.rb

@@ -1,8 +0,0 @@
-require 'test_helper'
-
-class IfTagUnitTest < Test::Unit::TestCase
-  def test_if_nodelist
-    template = Liquid::Template.parse('{% if true %}IF{% else %}ELSE{% endif %}')
-    assert_equal ['IF', 'ELSE'], template.root.nodelist[0].nodelist
-  end
-end

test/unit/template_unit_test.rb

@@ -1,69 +0,0 @@
-require 'test_helper'
-
-class TemplateUnitTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_sets_default_localization_in_document
-    t = Template.new
-    t.parse('')
-    assert_instance_of I18n, t.root.options[:locale]
-  end
-
-  def test_sets_default_localization_in_context_with_quick_initialization
-    t = Template.new
-    t.parse('{{foo}}', :locale => I18n.new(fixture("en_locale.yml")))
-
-    assert_instance_of I18n, t.root.options[:locale]
-    assert_equal fixture("en_locale.yml"), t.root.options[:locale].path
-  end
-
-  def test_with_cache_classes_tags_returns_the_same_class
-    original_cache_setting = Liquid.cache_classes
-    Liquid.cache_classes = true
-
-    original_klass = Class.new
-    Object.send(:const_set, :CustomTag, original_klass)
-    Template.register_tag('custom', CustomTag)
-
-    Object.send(:remove_const, :CustomTag)
-
-    new_klass = Class.new
-    Object.send(:const_set, :CustomTag, new_klass)
-
-    assert Template.tags['custom'].equal?(original_klass)
-  ensure
-    Object.send(:remove_const, :CustomTag)
-    Template.tags.delete('custom')
-    Liquid.cache_classes = original_cache_setting
-  end
-
-  def test_without_cache_classes_tags_reloads_the_class
-    original_cache_setting = Liquid.cache_classes
-    Liquid.cache_classes = false
-
-    original_klass = Class.new
-    Object.send(:const_set, :CustomTag, original_klass)
-    Template.register_tag('custom', CustomTag)
-
-    Object.send(:remove_const, :CustomTag)
-
-    new_klass = Class.new
-    Object.send(:const_set, :CustomTag, new_klass)
-
-    assert Template.tags['custom'].equal?(new_klass)
-  ensure
-    Object.send(:remove_const, :CustomTag)
-    Template.tags.delete('custom')
-    Liquid.cache_classes = original_cache_setting
-  end
-
-  class FakeTag; end
-
-  def test_tags_delete
-    Template.register_tag('fake', FakeTag)
-    assert_equal FakeTag, Template.tags['fake']
-
-    Template.tags.delete('fake')
-    assert_nil Template.tags['fake']
-  end
-end