Add convenience methods for getting a struct from a ruby object.

If we are trying to get the struct from something other than self, then we
should make sure to check the class of the object.  This util functions
make this easier.

.gitignore

@@ -6,3 +6,5 @@ pkg
 .rvmrc
 .ruby-version
 Gemfile.lock
+*.bundle
+/tmp

.travis.yml

@@ -1,8 +1,7 @@
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1
-  - 2.1.1
+  - 2.1.0
   - jruby-19mode
   - jruby-head
   - rbx-19mode
@@ -11,7 +10,7 @@ matrix:
     - rvm: rbx-19mode
     - rvm: jruby-head
 
-script: "gem build liquid.gemspec && gem install liquid-3.0.0.gem"
+script: "rake test"
 
 notifications:
   disable: true

History.md

@@ -3,10 +3,6 @@
 ## 3.0.0 / not yet released / branch "master"
 
 * ...
-* Add error messages for missing variables when :strict, see #352 [Daniel Gaiottino]
-* Properly set context rethrow_errors on render! #349 [Thierry Joyal, tjoyal]
-* Fix broken rendering of variables which are equal to false, see #345 [Florian Weingarten, fw42]
-* Remove ActionView template handler [Dylan Thacker-Smith, dylanahsmith]
 * Freeze lots of string literals for new Ruby 2.1 optimization, see #297 [Florian Weingarten, fw42]
 * Allow newlines in tags and variables, see #324 [Dylan Thacker-Smith, dylanahsmith]
 * Tag#parse is called after initialize, which now takes options instead of tokens as the 3rd argument. See #321 [Dylan Thacker-Smith, dylanahsmith]

README.md

@@ -1,6 +1,4 @@
 [![Build Status](https://secure.travis-ci.org/Shopify/liquid.png?branch=master)](http://travis-ci.org/Shopify/liquid)
-[![Inline docs](http://inch-ci.org/github/Shopify/liquid.png)](http://inch-ci.org/github/Shopify/liquid)
-
 # Liquid template engine
 
 * [Contributing guidelines](CONTRIBUTING.md)

Rakefile

@@ -1,5 +1,6 @@
 require 'rake'
 require 'rake/testtask'
+require 'rake/extensiontask'
 $LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
 require "liquid/version"
 
@@ -8,7 +9,7 @@ task :default => 'test'
 desc 'run test suite with default parser'
 Rake::TestTask.new(:base_test) do |t|
   t.libs << '.' << 'lib' << 'test'
-  t.test_files = FileList['test/{integration,unit}/**/*_test.rb']
+  t.test_files = FileList['test/liquid/**/*_test.rb']
   t.verbose = false
 end
 
@@ -75,3 +76,8 @@ desc "Run example"
 task :example do
   ruby "-w -d -Ilib example/server/server.rb"
 end
+
+Rake::ExtensionTask.new "liquid" do |ext|
+  ext.lib_dir = "lib/liquid"
+end
+Rake::Task[:test].prerequisites << :compile

ext/liquid/block.c

@@ -0,0 +1,168 @@
+#include "liquid_ext.h"
+
+VALUE cLiquidBlock;
+ID intern_assert_missing_delimitation, intern_block_delimiter, intern_is_blank, intern_new,
+   intern_new_with_options, intern_tags, intern_unknown_tag, intern_unterminated_tag,
+   intern_unterminated_variable;
+
+struct liquid_tag
+{
+    char *name, *markup;
+    long name_length, markup_length;
+};
+
+static bool parse_tag(struct liquid_tag *tag, char *token, long token_length)
+{
+    // Strip {{ and }} braces
+    token += 2;
+    token_length -= 4;
+
+    char *end = token + token_length;
+    while (token < end && isspace(*token))
+        token++;
+    tag->name = token;
+
+    char c = *token;
+    while (token < end && (isalnum(c) || c == '_'))
+        c = *(++token);
+    tag->name_length = token - tag->name;
+    if (!tag->name_length) {
+        memset(tag, 0, sizeof(*tag));
+        return false;
+    }
+
+    while (token < end && isspace(*token))
+        token++;
+    tag->markup = token;
+
+    char *last = end - 1;
+    while (token < last && isspace(*last))
+        last--;
+    end = last + 1;
+    tag->markup_length = end - token;
+    return true;
+}
+
+static VALUE rb_parse_body(VALUE self, VALUE tokenizerObj)
+{
+    struct liquid_tokenizer *tokenizer = LIQUID_TOKENIZER_GET_STRUCT(tokenizerObj);
+
+    bool blank = true;
+    VALUE nodelist = rb_iv_get(self, "@nodelist");
+    if (nodelist == Qnil) {
+        nodelist = rb_ary_new();
+        rb_iv_set(self, "@nodelist", nodelist);
+    } else {
+        rb_ary_clear(nodelist);
+    }
+
+    struct token token;
+    while (true) {
+        liquid_tokenizer_next(tokenizer, &token);
+        switch (token.type) {
+        case TOKEN_NONE:
+            /*
+             * Make sure that it's ok to end parsing in the current block.
+             * Effectively this method will throw an exception unless the current block is
+             * of type Document
+             */
+            rb_funcall(self, intern_assert_missing_delimitation, 0);
+            goto done;
+        case TOKEN_INVALID:
+        {
+            VALUE token_obj = rb_str_new(token.str, token.length);
+            if (token.str[1] == '%')
+                rb_funcall(self, intern_unterminated_tag, 1, token_obj);
+            else
+                rb_funcall(self, intern_unterminated_variable, 1, token_obj);
+            break;
+        }
+        case TOKEN_TAG:
+        {
+            struct liquid_tag tag;
+            if (!parse_tag(&tag, token.str, token.length)) {
+                // FIXME: provide more appropriate error message
+                rb_funcall(self, intern_unterminated_tag, 1, rb_str_new(token.str, token.length));
+            } else {
+                if (tag.name_length >= 3 && !memcmp(tag.name, "end", 3)) {
+                    VALUE block_delimiter = rb_funcall(self, intern_block_delimiter, 0);
+                    if (TYPE(block_delimiter) == T_STRING &&
+                        tag.name_length == RSTRING_LEN(block_delimiter) &&
+                        !memcmp(tag.name, RSTRING_PTR(block_delimiter), tag.name_length))
+                    {
+                        goto done;
+                    }
+                }
+
+                VALUE tags = rb_funcall(cLiquidTemplate, intern_tags, 0);
+                Check_Type(tags, T_HASH);
+                VALUE tag_name = rb_str_new(tag.name, tag.name_length);
+                VALUE tag_class = rb_hash_lookup(tags, tag_name);
+                VALUE markup = rb_str_new(tag.markup, tag.markup_length);
+                if (tag_class != Qnil) {
+                    VALUE options = rb_iv_get(self, "@options");
+                    if (options == Qnil)
+                        options = rb_hash_new();
+                    VALUE new_tag = rb_funcall(tag_class, intern_new_with_options, 4,
+                                               tag_name, markup, tokenizerObj, options);
+                    if (blank) {
+                        VALUE blank_block = rb_funcall(new_tag, intern_is_blank, 0);
+                        if (blank_block == Qnil || blank_block == Qfalse)
+                            blank = false;
+                    }
+                    rb_ary_push(nodelist, new_tag);
+                } else {
+                    rb_funcall(self, intern_unknown_tag, 3, tag_name, markup, tokenizerObj);
+                    /*
+                     * multi-block tags may store the nodelist in a block array on unknown_tag
+                     * then replace @nodelist with a new array. We need to use the new array
+                     * for the block following the tag token.
+                     */
+                    nodelist = rb_iv_get(self, "@nodelist");
+                }
+            }
+            break;
+        }
+        case TOKEN_VARIABLE:
+        {
+            VALUE markup = rb_str_new(token.str + 2, token.length - 4);
+            VALUE options = rb_iv_get(self, "@options");
+            VALUE new_var = rb_funcall(cLiquidVariable, intern_new, 2, markup, options);
+            rb_ary_push(nodelist, new_var);
+            blank = false;
+            break;
+        }
+        case TOKEN_STRING:
+            rb_ary_push(nodelist, rb_str_new(token.str, token.length));
+            if (blank) {
+                int i;
+                for (i = 0; i < token.length; i++) {
+                    if (!isspace(token.str[i])) {
+                        blank = false;
+                        break;
+                    }
+                }
+            }
+            break;
+        }
+    }
+done:
+    rb_iv_set(self, "@blank", blank ? Qtrue : Qfalse);
+    return Qnil;
+}
+
+void init_liquid_block()
+{
+    intern_assert_missing_delimitation = rb_intern("assert_missing_delimitation!");
+    intern_block_delimiter = rb_intern("block_delimiter");
+    intern_is_blank = rb_intern("blank?");
+    intern_new = rb_intern("new");
+    intern_new_with_options = rb_intern("new_with_options");
+    intern_tags = rb_intern("tags");
+    intern_unknown_tag = rb_intern("unknown_tag");
+    intern_unterminated_tag = rb_intern("unterminated_tag");
+    intern_unterminated_variable = rb_intern("unterminated_variable");
+
+    cLiquidBlock = rb_define_class_under(mLiquid, "Block", cLiquidTag);
+    rb_define_method(cLiquidBlock, "parse_body", rb_parse_body, 1);
+}

ext/liquid/block.h

@@ -0,0 +1,8 @@
+#ifndef LIQUID_BLOCK_H
+#define LIQUID_BLOCK_H
+
+void init_liquid_block();
+
+extern VALUE cLiquidBlock;
+
+#endif

ext/liquid/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+$CFLAGS << ' -Wall'
+create_makefile("liquid/liquid")

ext/liquid/liquid_ext.c

@@ -0,0 +1,15 @@
+#include "liquid_ext.h"
+
+VALUE mLiquid;
+VALUE cLiquidTemplate, cLiquidTag, cLiquidVariable;
+
+void Init_liquid(void)
+{
+    mLiquid = rb_define_module("Liquid");
+    cLiquidTemplate = rb_define_class_under(mLiquid, "Template", rb_cObject);
+    cLiquidTag = rb_define_class_under(mLiquid, "Tag", rb_cObject);
+    cLiquidVariable = rb_define_class_under(mLiquid, "Variable", rb_cObject);
+
+    init_liquid_tokenizer();
+    init_liquid_block();
+}

ext/liquid/liquid_ext.h

@@ -0,0 +1,15 @@
+#ifndef LIQUID_EXT_H
+#define LIQUID_EXT_H
+
+#include <stdbool.h>
+#include <ctype.h>
+#include <ruby.h>
+
+#include "tokenizer.h"
+#include "block.h"
+#include "utils.h"
+
+extern VALUE mLiquid;
+extern VALUE cLiquidTemplate, cLiquidTag, cLiquidVariable;
+
+#endif

ext/liquid/tokenizer.c

@@ -0,0 +1,113 @@
+#include "liquid_ext.h"
+
+VALUE cLiquidTokenizer;
+
+static void free_tokenizer(void *ptr)
+{
+    struct liquid_tokenizer *tokenizer = ptr;
+    xfree(tokenizer);
+}
+
+static VALUE rb_allocate(VALUE klass)
+{
+    VALUE obj;
+    struct liquid_tokenizer *tokenizer;
+
+    obj = Data_Make_Struct(klass, struct liquid_tokenizer, NULL, free_tokenizer, tokenizer);
+    return obj;
+}
+
+static VALUE rb_initialize(VALUE self, VALUE source)
+{
+    struct liquid_tokenizer *tokenizer;
+
+    Check_Type(source, T_STRING);
+    Data_Get_Struct(self, struct liquid_tokenizer, tokenizer);
+    tokenizer->cursor = RSTRING_PTR(source);
+    tokenizer->length = RSTRING_LEN(source);
+    return Qnil;
+}
+
+void liquid_tokenizer_next(struct liquid_tokenizer *tokenizer, struct token *token)
+{
+    if (tokenizer->length <= 0) {
+        memset(token, 0, sizeof(*token));
+        return;
+    }
+    token->type = TOKEN_STRING;
+
+    char *cursor = tokenizer->cursor;
+    char *last = tokenizer->cursor + tokenizer->length - 1;
+
+    while (cursor < last) {
+        if (*cursor++ != '{')
+            continue;
+
+        char c = *cursor++;
+        if (c != '%' && c != '{')
+            continue;
+        if (cursor - tokenizer->cursor > 2) {
+            token->type = TOKEN_STRING;
+            cursor -= 2;
+            goto found;
+        }
+        char *incomplete_end = cursor;
+        token->type = TOKEN_INVALID;
+        if (c == '%') {
+            while (cursor < last) {
+                if (*cursor++ != '%')
+                    continue;
+                c = *cursor++;
+                while (c == '%' && cursor <= last)
+                    c = *cursor++;
+                if (c != '}')
+                    continue;
+                token->type = TOKEN_TAG;
+                goto found;
+            }
+            cursor = incomplete_end;
+            goto found;
+        } else {
+            while (cursor < last) {
+                if (*cursor++ != '}')
+                    continue;
+                if (*cursor++ != '}') {
+                    incomplete_end = cursor - 1;
+                    continue;
+                }
+                token->type = TOKEN_VARIABLE;
+                goto found;
+            }
+            cursor = incomplete_end;
+            goto found;
+        }
+    }
+    cursor = last + 1;
+found:
+    token->str = tokenizer->cursor;
+    token->length = cursor - tokenizer->cursor;
+    tokenizer->cursor += token->length;
+    tokenizer->length -= token->length;
+}
+
+static VALUE rb_next(VALUE self)
+{
+    struct liquid_tokenizer *tokenizer;
+    Data_Get_Struct(self, struct liquid_tokenizer, tokenizer);
+
+    struct token token;
+    liquid_tokenizer_next(tokenizer, &token);
+    if (token.type == TOKEN_NONE)
+        return Qnil;
+
+    return rb_str_new(token.str, token.length);
+}
+
+void init_liquid_tokenizer()
+{
+    cLiquidTokenizer = rb_define_class_under(mLiquid, "Tokenizer", rb_cObject);
+    rb_define_alloc_func(cLiquidTokenizer, rb_allocate);
+    rb_define_method(cLiquidTokenizer, "initialize", rb_initialize, 1);
+    rb_define_method(cLiquidTokenizer, "next", rb_next, 0);
+    rb_define_alias(cLiquidTokenizer, "shift", "next");
+}

ext/liquid/tokenizer.h

@@ -0,0 +1,30 @@
+#ifndef LIQUID_TOKENIZER_H
+#define LIQUID_TOKENIZER_H
+
+extern VALUE cLiquidTokenizer;
+
+enum token_type {
+    TOKEN_NONE,
+    TOKEN_INVALID,
+    TOKEN_STRING,
+    TOKEN_TAG,
+    TOKEN_VARIABLE
+};
+
+struct token {
+    enum token_type type;
+    char *str;
+    int length;
+};
+
+struct liquid_tokenizer {
+    char *cursor;
+    int length;
+};
+
+void init_liquid_tokenizer();
+void liquid_tokenizer_next(struct liquid_tokenizer *tokenizer, struct token *token);
+
+#define LIQUID_TOKENIZER_GET_STRUCT(obj) ((struct liquid_tokenizer *)obj_get_data_ptr(obj, cLiquidTokenizer))
+
+#endif

ext/liquid/utils.c

@@ -0,0 +1,21 @@
+#include <ruby.h>
+
+void raise_type_error(VALUE expected, VALUE got)
+{
+    rb_raise(rb_eTypeError, "wrong argument type %s (expected %s)",
+                             rb_class2name(got), rb_class2name(expected));
+}
+
+void check_class(VALUE obj, int type, VALUE klass)
+{
+    Check_Type(obj, type);
+    VALUE obj_klass = RBASIC_CLASS(obj);
+    if (obj_klass != klass)
+        raise_type_error(klass, obj_klass);
+}
+
+void *obj_get_data_ptr(VALUE obj, VALUE klass)
+{
+    check_class(obj, T_DATA, klass);
+    return DATA_PTR(obj);
+}

ext/liquid/utils.h

@@ -0,0 +1,8 @@
+#ifndef LIQUID_UTILS_H
+#define LIQUID_UTILS_H
+
+void raise_type_error(VALUE expected, VALUE got);
+void check_class(VALUE klass);
+void *obj_get_data_ptr(VALUE obj, VALUE klass);
+
+#endif

init.rb

@@ -0,0 +1,8 @@
+require 'liquid'
+require 'extras/liquid_view'
+
+if defined? ActionView::Template and ActionView::Template.respond_to? :register_template_handler
+  ActionView::Template
+else
+  ActionView::Base
+end.register_template_handler(:liquid, LiquidView)

lib/extras/liquid_view.rb

@@ -0,0 +1,51 @@
+# LiquidView is a action view extension class. You can register it with rails
+# and use liquid as an template system for .liquid files
+#
+# Example
+#
+#   ActionView::Base::register_template_handler :liquid, LiquidView
+class LiquidView
+  PROTECTED_ASSIGNS = %w( template_root response _session template_class action_name request_origin session template
+                          _response url _request _cookies variables_added _flash params _headers request cookies
+                          ignore_missing_templates flash _params logger before_filter_chain_aborted headers )
+  PROTECTED_INSTANCE_VARIABLES = %w( @_request @controller @_first_render @_memoized__pick_template @view_paths
+                                     @helpers @assigns_added @template @_render_stack @template_format @assigns )
+
+  def self.call(template)
+    "LiquidView.new(self).render(template, local_assigns)"
+  end
+
+  def initialize(view)
+    @view = view
+  end
+
+  def render(template, local_assigns = nil)
+    @view.controller.headers["Content-Type"] ||= 'text/html; charset=utf-8'
+
+    # Rails 2.2 Template has source, but not locals
+    if template.respond_to?(:source) && !template.respond_to?(:locals)
+      assigns = (@view.instance_variables - PROTECTED_INSTANCE_VARIABLES).inject({}) do |hash, ivar|
+                  hash[ivar[1..-1]] = @view.instance_variable_get(ivar)
+                  hash
+                end
+    else
+      assigns = @view.assigns.reject{ |k,v| PROTECTED_ASSIGNS.include?(k) }
+    end
+
+    source = template.respond_to?(:source) ? template.source : template
+    local_assigns = (template.respond_to?(:locals) ? template.locals : local_assigns) || {}
+
+    if content_for_layout = @view.instance_variable_get("@content_for_layout")
+      assigns['content_for_layout'] = content_for_layout
+    end
+    assigns.merge!(local_assigns.stringify_keys)
+
+    liquid = Liquid::Template.parse(source)
+    liquid.render(assigns, :filters => [@view.controller.master_helper_module], :registers => {:action_view => @view, :controller => @view.controller})
+  end
+
+  def compilable?
+    false
+  end
+
+end

lib/liquid.rb

@@ -30,16 +30,13 @@ module Liquid
   VariableSegment             = /[\w\-]/
   VariableStart               = /\{\{/
   VariableEnd                 = /\}\}/
-  VariableIncompleteEnd       = /\}\}?/
   QuotedString                = /"[^"]*"|'[^']*'/
   QuotedFragment              = /#{QuotedString}|(?:[^\s,\|'"]|#{QuotedString})+/o
   TagAttributes               = /(\w+)\s*\:\s*(#{QuotedFragment})/o
-  AnyStartingTag              = /\{\{|\{\%/
-  PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om
-  TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/om
   VariableParser              = /\[[^\]]+\]|#{VariableSegment}+\??/o
 end
 
+require 'liquid/liquid'
 require "liquid/version"
 require 'liquid/lexer'
 require 'liquid/parser'

lib/liquid/block.rb

@@ -1,82 +1,26 @@
 module Liquid
   class Block < Tag
-    IsTag             = /\A#{TagStart}/o
-    IsVariable        = /\A#{VariableStart}/o
-    FullToken         = /\A#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om
-    ContentOfVariable = /\A#{VariableStart}(.*)#{VariableEnd}\z/om
+    def initialize(tag_name, markup, tokens)
+      super
+      parse_body(tokens)
+    end
 
     def blank?
       @blank || false
     end
 
-    def parse(tokens)
-      @blank = true
-      @nodelist ||= []
-      @nodelist.clear
-
-      # All child tags of the current block.
-      @children = []
-
-      while token = tokens.shift
-        case token
-        when IsTag
-          if token =~ FullToken
-
-            # if we found the proper block delimiter just end parsing here and let the outer block
-            # proceed
-            if block_delimiter == $1
-              end_tag
-              return
-            end
-
-            # fetch the tag from registered blocks
-            if tag = Template.tags[$1]
-              new_tag = tag.parse($1, $2, tokens, @options)
-              @blank &&= new_tag.blank?
-              @nodelist << new_tag
-              @children << new_tag
-            else
-              # this tag is not registered with the system
-              # pass it to the current block for special handling or error reporting
-              unknown_tag($1, $2, tokens)
-            end
-          else
-            raise SyntaxError.new(options[:locale].t("errors.syntax.tag_termination".freeze, :token => token, :tag_end => TagEnd.inspect))
-          end
-        when IsVariable
-          new_var = create_variable(token)
-          @nodelist << new_var
-          @children << new_var
-          @blank = false
-        when ''.freeze
-          # pass
-        else
-          @nodelist << token
-          @blank &&= (token =~ /\A\s*\z/)
-        end
-      end
-
-      # Make sure that it's ok to end parsing in the current block.
-      # Effectively this method will throw an exception unless the current block is
-      # of type Document
-      assert_missing_delimitation!
-    end
-
     # warnings of this block and all sub-tags
     def warnings
       all_warnings = []
       all_warnings.concat(@warnings) if @warnings
 
-      (@children || []).each do |node|
-        all_warnings.concat(node.warnings || [])
+      (nodelist || []).each do |node|
+        all_warnings.concat(node.warnings || []) if node.respond_to?(:warnings)
       end
 
       all_warnings
     end
 
-    def end_tag
-    end
-
     def unknown_tag(tag, params, tokens)
       case tag
       when 'else'.freeze
@@ -112,6 +56,14 @@ module Liquid
 
     protected
 
+    def unterminated_variable(token)
+      raise SyntaxError.new(options[:locale].t("errors.syntax.variable_termination".freeze, :token => token, :tag_end => VariableEnd.inspect))
+    end
+
+    def unterminated_tag(token)
+      raise SyntaxError.new(options[:locale].t("errors.syntax.tag_termination".freeze, :token => token, :tag_end => TagEnd.inspect))
+    end
+
     def assert_missing_delimitation!
       raise SyntaxError.new(options[:locale].t("errors.syntax.tag_never_closed".freeze, :block_name => block_name))
     end

lib/liquid/context.rb

@@ -15,8 +15,6 @@ module Liquid
   class Context
     attr_reader :scopes, :errors, :registers, :environments, :resource_limits
 
-    attr_accessor :rethrow_errors
-
     def initialize(environments = {}, outer_scope = {}, registers = {}, rethrow_errors = false, resource_limits = {})
       @environments    = [environments].flatten
       @scopes          = [(outer_scope || {})]
@@ -196,8 +194,7 @@ module Liquid
 
         if scope.nil?
           @environments.each do |e|
-            variable = lookup_and_evaluate(e, key)
-            unless variable.nil?
+            if variable = lookup_and_evaluate(e, key)
               scope = e
               break
             end
@@ -205,7 +202,6 @@ module Liquid
         end
 
         scope     ||= @environments.last || @scopes.last
-        handle_not_found(key) unless scope.has_key?(key)
         variable  ||= lookup_and_evaluate(scope, key)
 
         variable = variable.to_liquid
@@ -255,7 +251,6 @@ module Liquid
               # No key was present with the desired value and it wasn't one of the directly supported
               # keywords either. The only thing we got left is to return nil
             else
-              handle_not_found(markup)
               return nil
             end
 
@@ -285,10 +280,6 @@ module Liquid
           end
         end
       end # squash_instance_assigns_with_environments
-
-      def handle_not_found(variable)
-        @errors << "Variable {{#{variable}}} not found" if Template.error_mode == :strict
-      end
   end # Context
 
 end # Liquid

lib/liquid/document.rb

@@ -7,7 +7,7 @@ module Liquid
 
     # There isn't a real delimiter
     def block_delimiter
-      []
+      nil
     end
 
     # Document blocks don't need to be terminated since they are not actually opened

lib/liquid/standardfilters.rb

@@ -162,7 +162,7 @@ module Liquid
       input.to_s.gsub(/\n/, "<br />\n".freeze)
     end
 
-    # Reformat a date using Ruby's core Time#strftime( string ) -> string
+    # Reformat a date
     #
     #   %a - The abbreviated weekday name (``Sun'')
     #   %A - The  full  weekday  name (``Sunday'')
@@ -176,7 +176,6 @@ module Liquid
     #   %m - Month of the year (01..12)
     #   %M - Minute of the hour (00..59)
     #   %p - Meridian indicator (``AM''  or  ``PM'')
-    #   %s - Number of seconds since 1970-01-01 00:00:00 UTC.
     #   %S - Second of the minute (00..60)
     #   %U - Week  number  of the current year,
     #           starting with the first Sunday as the first
@@ -191,14 +190,34 @@ module Liquid
     #   %Y - Year with century
     #   %Z - Time zone name
     #   %% - Literal ``%'' character
-    #
-    #   See also: http://www.ruby-doc.org/core/Time.html#method-i-strftime
     def date(input, format)
-      return input if format.to_s.empty?
 
-      return input unless date = to_date(input)
+      if format.to_s.empty?
+        return input.to_s
+      end
 
-      date.strftime(format.to_s)
+      if ((input.is_a?(String) && !/\A\d+\z/.match(input.to_s).nil?) || input.is_a?(Integer)) && input.to_i > 0
+        input = Time.at(input.to_i)
+      end
+
+      date = if input.is_a?(String)
+        case input.downcase
+        when 'now'.freeze, 'today'.freeze
+          Time.now
+        else
+          Time.parse(input)
+        end
+      else
+        input
+      end
+
+      if date.respond_to?(:strftime)
+        date.strftime(format.to_s)
+      else
+        input
+      end
+    rescue
+      input
     end
 
     # Get the first element of the passed in array
@@ -243,21 +262,6 @@ module Liquid
       apply_operation(input, operand, :%)
     end
 
-    def round(input, n = 0)
-      result = to_number(input).round(to_number(n))
-      result = result.to_f if result.is_a?(BigDecimal)
-      result = result.to_i if n == 0
-      result
-    end
-
-    def ceil(input)
-      to_number(input).ceil.to_i
-    end
-
-    def floor(input)
-      to_number(input).floor.to_i
-    end
-
     def default(input, default_value = "".freeze)
       is_blank = input.respond_to?(:empty?) ? input.empty? : !input
       is_blank ? default_value : input
@@ -289,23 +293,6 @@ module Liquid
       end
     end
 
-    def to_date(obj)
-      return obj if obj.respond_to?(:strftime)
-
-      case obj
-      when 'now'.freeze, 'today'.freeze
-        Time.now
-      when /\A\d+\z/, Integer
-        Time.at(obj.to_i)
-      when String
-        Time.parse(obj)
-      else
-        nil
-      end
-    rescue ArgumentError
-      nil
-    end
-
     def apply_operation(input, operand, operation)
       result = to_number(input).send(operation, to_number(operand))
       result.is_a?(BigDecimal) ? result.to_f : result

lib/liquid/tag.rb

@@ -19,9 +19,6 @@ module Liquid
       @options    = options
     end
 
-    def parse(tokens)
-    end
-
     def name
       self.class.name.downcase
     end

lib/liquid/tags/ifchanged.rb

@@ -4,7 +4,7 @@ module Liquid
     def render(context)
       context.stack do
 
-        output = render_all(@nodelist, context)
+        output = super
 
         if output != context.registers[:ifchanged]
           context.registers[:ifchanged] = output

lib/liquid/tags/include.rb

@@ -35,9 +35,6 @@ module Liquid
       end
     end
 
-    def parse(tokens)
-    end
-
     def blank?
       false
     end

lib/liquid/tags/raw.rb

@@ -2,16 +2,13 @@ module Liquid
   class Raw < Block
     FullTokenPossiblyInvalid = /\A(.*)#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}\z/om
 
-    def parse(tokens)
+    def parse_body(tokens)
       @nodelist ||= []
       @nodelist.clear
       while token = tokens.shift
         if token =~ FullTokenPossiblyInvalid
           @nodelist << $1 if $1 != "".freeze
-          if block_delimiter == $2
-            end_tag
-            return
-          end
+          return if block_delimiter == $2
         end
         @nodelist << token if not token.empty?
       end

lib/liquid/tags/table_row.rb

@@ -54,7 +54,7 @@ module Liquid
 
           col += 1
 
-          result << "<td class=\"col#{col}\">" << render_all(@nodelist, context) << '</td>'
+          result << "<td class=\"col#{col}\">" << super << '</td>'
 
           if col == cols and (index != length - 1)
             col  = 0

lib/liquid/template.rb

@@ -22,12 +22,6 @@ module Liquid
     @@file_system = BlankFileSystem.new
 
     class << self
-      # Sets how strict the parser should be.
-      # :lax acts like liquid 2.5 and silently ignores malformed tags in most cases.
-      # :warn is the default and will give deprecation warnings when invalid syntax is used.
-      # :strict will enforce correct syntax.
-      attr_writer :error_mode
-
       def file_system
         @@file_system
       end
@@ -44,6 +38,14 @@ module Liquid
         @tags ||= {}
       end
 
+      # Sets how strict the parser should be.
+      # :lax acts like liquid 2.5 and silently ignores malformed tags in most cases.
+      # :warn is the default and will give deprecation warnings when invalid syntax is used.
+      # :strict will enforce correct syntax.
+      def error_mode=(mode)
+        @error_mode = mode
+      end
+
       def error_mode
         @error_mode || :lax
       end
@@ -58,6 +60,7 @@ module Liquid
       def parse(source, options = {})
         template = Template.new
         template.parse(source, options)
+        template
       end
     end
 
@@ -111,9 +114,7 @@ module Liquid
 
       context = case args.first
       when Liquid::Context
-        c = args.shift
-        c.rethrow_errors = true if @rethrow_errors
-        c
+        args.shift
       when Liquid::Drop
         drop = args.shift
         drop.context = Context.new([drop, assigns], instance_assigns, registers, @rethrow_errors, @resource_limits)
@@ -156,22 +157,14 @@ module Liquid
     end
 
     def render!(*args)
-      @rethrow_errors = true
-      render(*args)
+      @rethrow_errors = true; render(*args)
     end
 
     private
 
-    # Uses the <tt>Liquid::TemplateParser</tt> regexp to tokenize the passed source
     def tokenize(source)
       source = source.source if source.respond_to?(:source)
-      return [] if source.to_s.empty?
-      tokens = source.split(TemplateParser)
-
-      # removes the rogue empty element at the beginning of the array
-      tokens.shift if tokens[0] and tokens[0].empty?
-
-      tokens
+      Tokenizer.new(source.to_s)
     end
 
   end

liquid.gemspec

@@ -18,12 +18,17 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = ">= 1.3.7"
 
   s.test_files  = Dir.glob("{test}/**/*")
-  s.files       = Dir.glob("{lib}/**/*") + %w(MIT-LICENSE README.md)
+  s.files       = Dir.glob("{lib,ext}/**/*") + %w(MIT-LICENSE README.md)
+  s.extensions  = ['ext/liquid/extconf.rb']
 
   s.extra_rdoc_files  = ["History.md", "README.md"]
 
   s.require_path = "lib"
 
-  s.add_development_dependency 'stackprof' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0")
   s.add_development_dependency 'rake'
+  s.add_development_dependency 'activesupport'
+  if RUBY_ENGINE == 'ruby'
+    s.add_development_dependency 'rake-compiler'
+    s.add_development_dependency 'stackprof' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0")
+  end
 end

performance/benchmark.rb

@@ -1,3 +1,4 @@
+require 'rubygems'
 require 'benchmark'
 require File.dirname(__FILE__) + '/theme_runner'
 

performance/shopify/database.rb

@@ -1,6 +1,6 @@
 require 'yaml'
-
 module Database
+
   # Load the standard vision toolkit database and re-arrage it to be simply exportable
   # to liquid as assigns. All this is based on Shopify
   def self.tables

performance/shopify/json_filter.rb

@@ -1,9 +1,7 @@
-require 'json'
-
 module JsonFilter
 
   def json(object)
-    JSON.dump(object.reject {|k,v| k == "collections" })
+    object.reject {|k,v| k == "collections" }.to_json
   end
 
 end

performance/theme_runner.rb

@@ -6,6 +6,11 @@
 # Shopify which is likely the biggest user of liquid in the world which something to the tune of several
 # million Template#render calls a day.
 
+require 'rubygems'
+require 'active_support'
+require 'active_support/json'
+require 'yaml'
+require 'digest/md5'
 require File.dirname(__FILE__) + '/shopify/liquid'
 require File.dirname(__FILE__) + '/shopify/database.rb'
 
@@ -76,3 +81,6 @@ class ThemeRunner
     end
   end
 end
+
+
+

test/integration/context_test.rb

@@ -1,23 +0,0 @@
-require 'test_helper'
-
-class ContextTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_override_global_filter
-    global = Module.new do
-      def notice(output)
-        "Global #{output}"
-      end
-    end
-
-    local = Module.new do
-      def notice(output)
-        "Local #{output}"
-      end
-    end
-
-    Template.register_filter(global)
-    assert_equal 'Global test', Template.parse("{{'test' | notice }}").render!
-    assert_equal 'Local test', Template.parse("{{'test' | notice }}").render!({}, :filters => [local])
-  end
-end

test/integration/variable_test.rb

@@ -1,72 +0,0 @@
-require 'test_helper'
-
-class VariableTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_simple_variable
-    template = Template.parse(%|{{test}}|)
-    assert_equal 'worked', template.render!('test' => 'worked')
-    assert_equal 'worked wonderfully', template.render!('test' => 'worked wonderfully')
-  end
-
-  def test_simple_with_whitespaces
-    template = Template.parse(%|  {{ test }}  |)
-    assert_equal '  worked  ', template.render!('test' => 'worked')
-    assert_equal '  worked wonderfully  ', template.render!('test' => 'worked wonderfully')
-  end
-
-  def test_ignore_unknown
-    template = Template.parse(%|{{ test }}|)
-    assert_equal '', template.render!
-  end
-
-  def test_hash_scoping
-    template = Template.parse(%|{{ test.test }}|)
-    assert_equal 'worked', template.render!('test' => {'test' => 'worked'})
-  end
-
-  def test_false_renders_as_false
-    assert_equal 'false', Template.parse("{{ foo }}").render!('foo' => false)
-  end
-
-  def test_preset_assigns
-    template = Template.parse(%|{{ test }}|)
-    template.assigns['test'] = 'worked'
-    assert_equal 'worked', template.render!
-  end
-
-  def test_reuse_parsed_template
-    template = Template.parse(%|{{ greeting }} {{ name }}|)
-    template.assigns['greeting'] = 'Goodbye'
-    assert_equal 'Hello Tobi', template.render!('greeting' => 'Hello', 'name' => 'Tobi')
-    assert_equal 'Hello ', template.render!('greeting' => 'Hello', 'unknown' => 'Tobi')
-    assert_equal 'Hello Brian', template.render!('greeting' => 'Hello', 'name' => 'Brian')
-    assert_equal 'Goodbye Brian', template.render!('name' => 'Brian')
-    assert_equal({'greeting'=>'Goodbye'}, template.assigns)
-  end
-
-  def test_assigns_not_polluted_from_template
-    template = Template.parse(%|{{ test }}{% assign test = 'bar' %}{{ test }}|)
-    template.assigns['test'] = 'baz'
-    assert_equal 'bazbar', template.render!
-    assert_equal 'bazbar', template.render!
-    assert_equal 'foobar', template.render!('test' => 'foo')
-    assert_equal 'bazbar', template.render!
-  end
-
-  def test_hash_with_default_proc
-    template = Template.parse(%|Hello {{ test }}|)
-    assigns = Hash.new { |h,k| raise "Unknown variable '#{k}'" }
-    assigns['test'] = 'Tobi'
-    assert_equal 'Hello Tobi', template.render!(assigns)
-    assigns.delete('test')
-    e = assert_raises(RuntimeError) {
-      template.render!(assigns)
-    }
-    assert_equal "Unknown variable 'test'", e.message
-  end
-
-  def test_multiline_variable
-    assert_equal 'worked', Template.parse("{{\ntest\n}}").render!('test' => 'worked')
-  end
-end

test/liquid/block_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class BlockUnitTest < Test::Unit::TestCase
+class BlockTest < Test::Unit::TestCase
   include Liquid
 
   def test_blankspace

test/liquid/condition_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class ConditionUnitTest < Test::Unit::TestCase
+class ConditionTest < Test::Unit::TestCase
   include Liquid
 
   def test_basic_condition

test/liquid/context_test.rb

@@ -63,7 +63,7 @@ class ArrayLike
   end
 end
 
-class ContextUnitTest < Test::Unit::TestCase
+class ContextTest < Test::Unit::TestCase
   include Liquid
 
   def setup
@@ -162,6 +162,24 @@ class ContextUnitTest < Test::Unit::TestCase
 
   end
 
+  def test_override_global_filter
+    global = Module.new do
+      def notice(output)
+        "Global #{output}"
+      end
+    end
+
+    local = Module.new do
+      def notice(output)
+        "Local #{output}"
+      end
+    end
+
+    Template.register_filter(global)
+    assert_equal 'Global test', Template.parse("{{'test' | notice }}").render!
+    assert_equal 'Local test', Template.parse("{{'test' | notice }}").render!({}, :filters => [local])
+  end
+
   def test_only_intended_filters_make_it_there
 
     filter = Module.new do
@@ -457,22 +475,4 @@ class ContextUnitTest < Test::Unit::TestCase
     assert_kind_of CategoryDrop, @context['category']
     assert_equal @context, @context['category'].context
   end
-
-  def test_strict_variables_not_found
-    with_error_mode(:strict) do
-      @context['does_not_exist']
-      assert(@context.errors.length == 1)
-      assert_equal(@context.errors[0], 'Variable {{does_not_exist}} not found')
-    end
-  end
-
-  def test_strict_nested_variables_not_found
-    with_error_mode(:strict) do
-      @context['hash'] = {'this' => 'exists'}
-      @context['hash.does_not_exist']
-      assert(@context.errors.length == 1)
-      assert_equal(@context.errors[0], 'Variable {{hash.does_not_exist}} not found')
-    end
-  end
-
 end # ContextTest

test/liquid/file_system_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class FileSystemUnitTest < Test::Unit::TestCase
+class FileSystemTest < Test::Unit::TestCase
   include Liquid
 
   def test_default

test/liquid/i18n_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class I18nUnitTest < Test::Unit::TestCase
+class I18nTest < Test::Unit::TestCase
   include Liquid
 
   def setup

test/liquid/lexer_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class LexerUnitTest < Test::Unit::TestCase
+class LexerTest < Test::Unit::TestCase
   include Liquid
 
   def test_strings

test/liquid/module_ex_test.rb

@@ -36,7 +36,7 @@ class TestClassC::LiquidDropClass
   end
 end
 
-class ModuleExUnitTest < Test::Unit::TestCase
+class ModuleExTest < Test::Unit::TestCase
   include Liquid
 
   def setup

test/liquid/parser_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class ParserUnitTest < Test::Unit::TestCase
+class ParserTest < Test::Unit::TestCase
   include Liquid
 
   def test_consume

test/liquid/parsing_quirks_test.rb

@@ -3,9 +3,12 @@ require 'test_helper'
 class ParsingQuirksTest < Test::Unit::TestCase
   include Liquid
 
-  def test_parsing_css
-    text = " div { font-weight: bold; } "
-    assert_equal text, Template.parse(text).render!
+  def test_error_with_css
+    text = %| div { font-weight: bold; } |
+    template = Template.parse(text)
+
+    assert_equal text, template.render!
+    assert_equal [String], template.root.nodelist.collect {|i| i.class}
   end
 
   def test_raise_on_single_close_bracet

test/liquid/regexp_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class RegexpUnitTest < Test::Unit::TestCase
+class RegexpTest < Test::Unit::TestCase
   include Liquid
 
   def test_empty

test/liquid/security_test.rb

@@ -54,10 +54,10 @@ class SecurityTest < Test::Unit::TestCase
   def test_does_not_add_drop_methods_to_symbol_table
     current_symbols = Symbol.all_symbols
 
-    assigns = { 'drop' => Drop.new }
-    assert_equal "", Template.parse("{{ drop.custom_method_1 }}", assigns).render!
-    assert_equal "", Template.parse("{{ drop.custom_method_2 }}", assigns).render!
-    assert_equal "", Template.parse("{{ drop.custom_method_3 }}", assigns).render!
+    drop = Drop.new
+    drop.invoke_drop("custom_method_1")
+    drop.invoke_drop("custom_method_2")
+    drop.invoke_drop("custom_method_3")
 
     assert_equal [], (Symbol.all_symbols - current_symbols)
   end

test/liquid/standard_filter_test.rb

@@ -115,13 +115,6 @@ class StandardFiltersTest < Test::Unit::TestCase
     assert_equal [{"a" => 1}, {"a" => 2}, {"a" => 3}, {"a" => 4}], @filters.sort([{"a" => 4}, {"a" => 3}, {"a" => 1}, {"a" => 2}], "a")
   end
 
-  def test_numerical_vs_lexicographical_sort
-    assert_equal [2, 10], @filters.sort([10, 2])
-    assert_equal [{"a" => 2}, {"a" => 10}], @filters.sort([{"a" => 10}, {"a" => 2}], "a")
-    assert_equal ["10", "2"], @filters.sort(["10", "2"])
-    assert_equal [{"a" => "10"}, {"a" => "2"}], @filters.sort([{"a" => "10"}, {"a" => "2"}], "a")
-  end
-
   def test_reverse
     assert_equal [4,3,2,1], @filters.reverse([1,2,3,4])
   end
@@ -193,6 +186,7 @@ class StandardFiltersTest < Test::Unit::TestCase
     assert_equal "07/05/2006", @filters.date("1152098955", "%m/%d/%Y")
   end
 
+
   def test_first_last
     assert_equal 1, @filters.first([1,2,3])
     assert_equal 3, @filters.last([1,2,3])
@@ -273,22 +267,6 @@ class StandardFiltersTest < Test::Unit::TestCase
     assert_template_result "1", "{{ 3 | modulo:2 }}"
   end
 
-  def test_round
-    assert_template_result "5", "{{ input | round }}", 'input' => 4.6
-    assert_template_result "4", "{{ '4.3' | round }}"
-    assert_template_result "4.56", "{{ input | round: 2 }}", 'input' => 4.5612
-  end
-
-  def test_ceil
-    assert_template_result "5", "{{ input | ceil }}", 'input' => 4.6
-    assert_template_result "5", "{{ '4.3' | ceil }}"
-  end
-
-  def test_floor
-    assert_template_result "4", "{{ input | floor }}", 'input' => 4.6
-    assert_template_result "4", "{{ '4.3' | floor }}"
-  end
-
   def test_append
     assigns = {'a' => 'bc', 'b' => 'd' }
     assert_template_result('bcd',"{{ a | append: 'd'}}",assigns)

test/liquid/strainer_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class StrainerUnitTest < Test::Unit::TestCase
+class StrainerTest < Test::Unit::TestCase
   include Liquid
 
   module AccessScopeFilters

test/liquid/tags/case_tag_test.rb

@@ -1,10 +1,10 @@
 require 'test_helper'
 
-class CaseTagUnitTest < Test::Unit::TestCase
+class CaseTagTest < Test::Unit::TestCase
   include Liquid
 
   def test_case_nodelist
     template = Liquid::Template.parse('{% case var %}{% when true %}WHEN{% else %}ELSE{% endcase %}')
     assert_equal ['WHEN', 'ELSE'], template.root.nodelist[0].nodelist
   end
-end
+end # CaseTest

test/liquid/tags/for_tag_test.rb

@@ -1,11 +1,5 @@
 require 'test_helper'
 
-class ThingWithValue < Liquid::Drop
-  def value
-    3
-  end
-end
-
 class ForTagTest < Test::Unit::TestCase
   include Liquid
 
@@ -40,20 +34,6 @@ HERE
     assert_template_result(' 1  2  3 ','{%for item in (1..3) %} {{item}} {%endfor%}')
   end
 
-  def test_for_with_variable_range
-    assert_template_result(' 1  2  3 ','{%for item in (1..foobar) %} {{item}} {%endfor%}', "foobar" => 3)
-  end
-
-  def test_for_with_hash_value_range
-    foobar = { "value" => 3 }
-    assert_template_result(' 1  2  3 ','{%for item in (1..foobar.value) %} {{item}} {%endfor%}', "foobar" => foobar)
-  end
-
-  def test_for_with_drop_value_range
-    foobar = ThingWithValue.new
-    assert_template_result(' 1  2  3 ','{%for item in (1..foobar.value) %} {{item}} {%endfor%}', "foobar" => foobar)
-  end
-
   def test_for_with_variable
     assert_template_result(' 1  2  3 ','{%for item in array%} {{item}} {%endfor%}','array' => [1,2,3])
     assert_template_result('123','{%for item in array%}{{item}}{%endfor%}','array' => [1,2,3])
@@ -315,6 +295,16 @@ HERE
     assert_template_result(expected, template, assigns)
   end
 
+  def test_for_nodelist
+    template = Liquid::Template.parse('{% for item in items %}FOR{% endfor %}')
+    assert_equal ['FOR'], template.root.nodelist[0].nodelist
+  end
+
+  def test_for_else_nodelist
+    template = Liquid::Template.parse('{% for item in items %}FOR{% else %}ELSE{% endfor %}')
+    assert_equal ['FOR', 'ELSE'], template.root.nodelist[0].nodelist
+  end
+
   class LoaderDrop < Liquid::Drop
     attr_accessor :each_called, :load_slice_called
 

test/liquid/tags/if_else_tag_test.rb

@@ -158,9 +158,14 @@ class IfElseTagTest < Test::Unit::TestCase
                            %({% if 'gnomeslab-and-or-liquid' contains 'gnomeslab-and-or-liquid' %}yes{% endif %}))
   end
 
+  def test_if_nodelist
+    template = Liquid::Template.parse('{% if true %}IF{% else %}ELSE{% endif %}')
+    assert_equal ['IF', 'ELSE'], template.root.nodelist[0].nodelist
+  end
+
   def test_operators_are_whitelisted
     assert_raise(SyntaxError) do
       assert_template_result('', %({% if 1 or throw or or 1 %}yes{% endif %}))
     end
   end
-end
+end # IfElseTest

test/liquid/tags/standard_tag_test.rb

@@ -3,6 +3,12 @@ require 'test_helper'
 class StandardTagTest < Test::Unit::TestCase
   include Liquid
 
+  def test_tag
+    tag = Tag.parse('tag', [], [], {})
+    assert_equal 'liquid::tag', tag.name
+    assert_equal '', tag.render(Context.new)
+  end
+
   def test_no_transform
     assert_template_result('this text should come out of the template without change...',
                            'this text should come out of the template without change...')

test/liquid/template_test.rb

@@ -22,12 +22,6 @@ class SomethingWithLength
   liquid_methods :length
 end
 
-class ErroneousDrop < Liquid::Drop
-  def bad_method
-    raise 'ruby error in drop'
-  end
-end
-
 class TemplateTest < Test::Unit::TestCase
   include Liquid
 
@@ -144,13 +138,17 @@ class TemplateTest < Test::Unit::TestCase
     assert_equal 'haha', t.parse("{{baz}}").render!(drop)
   end
 
-  def test_render_bang_force_rethrow_errors_on_passed_context
-    context = Context.new({'drop' => ErroneousDrop.new})
-    t = Template.new.parse('{{ drop.bad_method }}')
+  def test_sets_default_localization_in_document
+    t = Template.new
+    t.parse('')
+    assert_instance_of I18n, t.root.options[:locale]
+  end
 
-    e = assert_raises RuntimeError do
-      t.render!(context)
-    end
-    assert_equal 'ruby error in drop', e.message
+  def test_sets_default_localization_in_context_with_quick_initialization
+    t = Template.new
+    t.parse('{{foo}}', :locale => I18n.new(fixture("en_locale.yml")))
+
+    assert_instance_of I18n, t.root.options[:locale]
+    assert_equal fixture("en_locale.yml"), t.root.options[:locale].path
   end
 end

test/liquid/tokenizer_test.rb

@@ -0,0 +1,64 @@
+require 'test_helper'
+
+class TokenizerTest < Test::Unit::TestCase
+  def test_tokenize_strings
+    assert_equal [' '], tokenize(' ')
+    assert_equal ['hello world'], tokenize('hello world')
+  end
+
+  def test_tokenize_variables
+    assert_equal ['{{funk}}'], tokenize('{{funk}}')
+    assert_equal [' ', '{{funk}}', ' '], tokenize(' {{funk}} ')
+    assert_equal [' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], tokenize(' {{funk}} {{so}} {{brother}} ')
+    assert_equal [' ', '{{  funk  }}', ' '], tokenize(' {{  funk  }} ')
+  end
+
+  def test_tokenize_blocks
+    assert_equal ['{%comment%}'], tokenize('{%comment%}')
+    assert_equal [' ', '{%comment%}', ' '], tokenize(' {%comment%} ')
+
+    assert_equal [' ', '{%comment%}', ' ', '{%endcomment%}', ' '], tokenize(' {%comment%} {%endcomment%} ')
+    assert_equal ['  ', '{% comment %}', ' ', '{% endcomment %}', ' '], tokenize("  {% comment %} {% endcomment %} ")
+  end
+
+  def test_tokenize_incomplete_end
+    assert_tokens 'before{{ incomplete }after', ['before', '{{ incomplete }', 'after']
+    assert_tokens 'before{% incomplete %after', ['before', '{%', ' incomplete %after']
+  end
+
+  def test_tokenize_no_end
+    assert_tokens 'before{{ unterminated ', ['before', '{{', ' unterminated ']
+    assert_tokens 'before{% unterminated ', ['before', '{%', ' unterminated ']
+  end
+
+  private
+
+  def assert_tokens(source, expected)
+    assert_equal expected, tokenize(source)
+    assert_equal expected, old_tokenize(source)
+  end
+
+  def tokenize(source)
+    tokenizer = Liquid::Tokenizer.new(source)
+    tokens = []
+    while token = tokenizer.next
+      tokens << token
+    end
+    tokens
+  end
+
+  AnyStartingTag        = /\{\{|\{\%/
+  VariableIncompleteEnd = /\}\}?/
+  PartialTemplateParser = /#{Liquid::TagStart}.*?#{Liquid::TagEnd}|#{Liquid::VariableStart}.*?#{VariableIncompleteEnd}/o
+  TemplateParser        = /(#{PartialTemplateParser}|#{AnyStartingTag})/o
+
+  def old_tokenize(source)
+    return [] if source.to_s.empty?
+    tokens = source.split(TemplateParser)
+
+    # removes the rogue empty element at the beginning of the array
+    tokens.shift if tokens[0] and tokens[0].empty?
+
+    tokens
+  end
+end

test/liquid/variable_test.rb

@@ -1,6 +1,6 @@
 require 'test_helper'
 
-class VariableUnitTest < Test::Unit::TestCase
+class VariableTest < Test::Unit::TestCase
   include Liquid
 
   def test_variable
@@ -134,3 +134,71 @@ class VariableUnitTest < Test::Unit::TestCase
     end
   end
 end
+
+
+class VariableResolutionTest < Test::Unit::TestCase
+  include Liquid
+
+  def test_simple_variable
+    template = Template.parse(%|{{test}}|)
+    assert_equal 'worked', template.render!('test' => 'worked')
+    assert_equal 'worked wonderfully', template.render!('test' => 'worked wonderfully')
+  end
+
+  def test_simple_with_whitespaces
+    template = Template.parse(%|  {{ test }}  |)
+    assert_equal '  worked  ', template.render!('test' => 'worked')
+    assert_equal '  worked wonderfully  ', template.render!('test' => 'worked wonderfully')
+  end
+
+  def test_ignore_unknown
+    template = Template.parse(%|{{ test }}|)
+    assert_equal '', template.render!
+  end
+
+  def test_hash_scoping
+    template = Template.parse(%|{{ test.test }}|)
+    assert_equal 'worked', template.render!('test' => {'test' => 'worked'})
+  end
+
+  def test_preset_assigns
+    template = Template.parse(%|{{ test }}|)
+    template.assigns['test'] = 'worked'
+    assert_equal 'worked', template.render!
+  end
+
+  def test_reuse_parsed_template
+    template = Template.parse(%|{{ greeting }} {{ name }}|)
+    template.assigns['greeting'] = 'Goodbye'
+    assert_equal 'Hello Tobi', template.render!('greeting' => 'Hello', 'name' => 'Tobi')
+    assert_equal 'Hello ', template.render!('greeting' => 'Hello', 'unknown' => 'Tobi')
+    assert_equal 'Hello Brian', template.render!('greeting' => 'Hello', 'name' => 'Brian')
+    assert_equal 'Goodbye Brian', template.render!('name' => 'Brian')
+    assert_equal({'greeting'=>'Goodbye'}, template.assigns)
+  end
+
+  def test_assigns_not_polluted_from_template
+    template = Template.parse(%|{{ test }}{% assign test = 'bar' %}{{ test }}|)
+    template.assigns['test'] = 'baz'
+    assert_equal 'bazbar', template.render!
+    assert_equal 'bazbar', template.render!
+    assert_equal 'foobar', template.render!('test' => 'foo')
+    assert_equal 'bazbar', template.render!
+  end
+
+  def test_hash_with_default_proc
+    template = Template.parse(%|Hello {{ test }}|)
+    assigns = Hash.new { |h,k| raise "Unknown variable '#{k}'" }
+    assigns['test'] = 'Tobi'
+    assert_equal 'Hello Tobi', template.render!(assigns)
+    assigns.delete('test')
+    e = assert_raises(RuntimeError) {
+      template.render!(assigns)
+    }
+    assert_equal "Unknown variable 'test'", e.message
+  end
+
+  def test_multiline_variable
+    assert_equal 'worked', Template.parse("{{\ntest\n}}").render!('test' => 'worked')
+  end
+end # VariableTest

test/unit/tag_unit_test.rb

@@ -1,11 +0,0 @@
-require 'test_helper'
-
-class TagUnitTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_tag
-    tag = Tag.parse('tag', [], [], {})
-    assert_equal 'liquid::tag', tag.name
-    assert_equal '', tag.render(Context.new)
-  end
-end

test/unit/tags/for_tag_unit_test.rb

@@ -1,13 +0,0 @@
-require 'test_helper'
-
-class ForTagUnitTest < Test::Unit::TestCase
-  def test_for_nodelist
-    template = Liquid::Template.parse('{% for item in items %}FOR{% endfor %}')
-    assert_equal ['FOR'], template.root.nodelist[0].nodelist
-  end
-
-  def test_for_else_nodelist
-    template = Liquid::Template.parse('{% for item in items %}FOR{% else %}ELSE{% endfor %}')
-    assert_equal ['FOR', 'ELSE'], template.root.nodelist[0].nodelist
-  end
-end

test/unit/tags/if_tag_unit_test.rb

@@ -1,8 +0,0 @@
-require 'test_helper'
-
-class IfTagUnitTest < Test::Unit::TestCase
-  def test_if_nodelist
-    template = Liquid::Template.parse('{% if true %}IF{% else %}ELSE{% endif %}')
-    assert_equal ['IF', 'ELSE'], template.root.nodelist[0].nodelist
-  end
-end

test/unit/template_unit_test.rb

@@ -1,19 +0,0 @@
-require 'test_helper'
-
-class TemplateUnitTest < Test::Unit::TestCase
-  include Liquid
-
-  def test_sets_default_localization_in_document
-    t = Template.new
-    t.parse('')
-    assert_instance_of I18n, t.root.options[:locale]
-  end
-
-  def test_sets_default_localization_in_context_with_quick_initialization
-    t = Template.new
-    t.parse('{{foo}}', :locale => I18n.new(fixture("en_locale.yml")))
-
-    assert_instance_of I18n, t.root.options[:locale]
-    assert_equal fixture("en_locale.yml"), t.root.options[:locale].path
-  end
-end

test/unit/tokenizer_unit_test.rb

@@ -1,29 +0,0 @@
-require 'test_helper'
-
-class TokenizerTest < Test::Unit::TestCase
-  def test_tokenize_strings
-    assert_equal [' '], tokenize(' ')
-    assert_equal ['hello world'], tokenize('hello world')
-  end
-
-  def test_tokenize_variables
-    assert_equal ['{{funk}}'], tokenize('{{funk}}')
-    assert_equal [' ', '{{funk}}', ' '], tokenize(' {{funk}} ')
-    assert_equal [' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], tokenize(' {{funk}} {{so}} {{brother}} ')
-    assert_equal [' ', '{{  funk  }}', ' '], tokenize(' {{  funk  }} ')
-  end
-
-  def test_tokenize_blocks
-    assert_equal ['{%comment%}'], tokenize('{%comment%}')
-    assert_equal [' ', '{%comment%}', ' '], tokenize(' {%comment%} ')
-
-    assert_equal [' ', '{%comment%}', ' ', '{%endcomment%}', ' '], tokenize(' {%comment%} {%endcomment%} ')
-    assert_equal ['  ', '{% comment %}', ' ', '{% endcomment %}', ' '], tokenize("  {% comment %} {% endcomment %} ")
-  end
-
-  private
-
-  def tokenize(source)
-    Liquid::Template.new.send(:tokenize, source)
-  end
-end