cache variable lookups

Liquid UTF-8 support

.gitignore

@@ -4,7 +4,3 @@
 pkg
 *.rbc
 .rvmrc
-*.o
-*.bundle
-ext/liquid/Makefile
-

History.md

@@ -1,5 +1,13 @@
 # Liquid Version History
 
+## 2.5.0 / 2013-03-06
+
+* Prevent Object methods from being called on drops
+* Avoid symbol injection from liquid
+* Added break and continue statements
+* Fix filter parser for args without space separators
+* Add support for filter keyword arguments
+
 ## 2.4.0 / 2012-08-03
 
 * Performance improvements

Rakefile

@@ -2,50 +2,12 @@
 
 require 'rubygems'
 require 'rake'
-require 'rake/clean'
-require 'fileutils'
 require 'rake/testtask'
 require 'rubygems/package_task'
 
-task :default => [:compile, :test]
+task :default => 'test'
 
-task :gen do
-  sh "leg -oext/liquid/liquid_context.c ext/liquid/liquid_context.leg"
-end
-
-task :compile => [:gen, :liquid_ext]
-
-extension = "liquid_ext"
-ext = "ext/liquid"
-ext_so = "#{ext}/#{extension}.#{RbConfig::CONFIG['DLEXT']}"
-ext_files = FileList[
-  "#{ext}/*.c",
-  "#{ext}/*.h",
-  "#{ext}/*.leg",
-  "#{ext}/extconf.rb",
-  "#{ext}/Makefile",
-  "lib"
-]
-
-task "lib" do
-  directory "lib"
-end
-
-desc "Builds just the #{extension} extension"
-task extension.to_sym => [:gen, "#{ext}/Makefile", ext_so ]
-
-file "#{ext}/Makefile" => ["#{ext}/extconf.rb"] do
-  Dir.chdir(ext) do ruby "extconf.rb" end
-end
-
-file ext_so => ext_files do
-  Dir.chdir(ext) do
-    sh "make"
-  end
-  cp ext_so, "lib"
-end
-
-Rake::TestTask.new(:test => [:gen, 'liquid_ext']) do |t|
+Rake::TestTask.new(:test) do |t|
   t.libs << '.' << 'lib' << 'test'
   t.test_files = FileList['test/liquid/**/*_test.rb']
   t.verbose = false
@@ -80,7 +42,7 @@ namespace :profile do
 
   desc "Run KCacheGrind"
   task :grind => :run  do
-    system "qcachegrind /tmp//callgrind.liquid.txt"
+    system "qcachegrind /tmp/liquid.rubyprof_calltreeprinter.txt"
   end
 
 end

ext/liquid/extconf.rb

@@ -1,6 +0,0 @@
-require 'mkmf'
-
-dir_config("liquid_ext")
-have_library("c", "main")
-
-create_makefile("liquid_ext")

ext/liquid/liquid_context.c

@@ -1,764 +0,0 @@
-/* A recursive-descent parser generated by peg 0.1.9 */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#define YYRULECOUNT 14
- 
-//uncomment to get more debug instrumentation
-//
-//#define YY_DEBUG 
-
-#include <ruby.h>
-
-#define EMIT(sym, data) \
-  rb_ary_push(ctx->rb_tokens, rb_ary_new3(2, ID2SYM(rb_intern(sym)), data)); 
-
-#define yy_rb_str rb_str_new(yytext, yyleng)
-
-#define YYSTYPE VALUE
-#define YY_CTX_LOCAL
-#define YY_CTX_MEMBERS VALUE rb_tokens; char *p; int p_len;
-
-#define YY_INPUT(buf, result, max_size) { \
-    result = ctx->p_len;  \
-    if (result>0 || EOF == ctx->p[0]) { \
-      if (max_size < result) { result = max_size; } \
-      strncpy(buf, ctx->p, result); \
-      buf[result] = '\0'; \
-      yyprintf((stderr, "\nREFILLING %d chars now:<%s>", result, buf));      \
-      ctx->p += result; ctx->p_len -= result; \
-      yyprintf((stderr, "\nREFILLING DONE size left: %d <%s>", ctx->p_len, ctx->p));      \
-    } \
-  }
-
-#ifndef YY_LOCAL
-#define YY_LOCAL(T)	static T
-#endif
-#ifndef YY_ACTION
-#define YY_ACTION(T)	static T
-#endif
-#ifndef YY_RULE
-#define YY_RULE(T)	static T
-#endif
-#ifndef YY_PARSE
-#define YY_PARSE(T)	T
-#endif
-#ifndef YYPARSE
-#define YYPARSE		yyparse
-#endif
-#ifndef YYPARSEFROM
-#define YYPARSEFROM	yyparsefrom
-#endif
-#ifndef YY_INPUT
-#define YY_INPUT(buf, result, max_size)			\
-  {							\
-    int yyc= getchar();					\
-    result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1);	\
-    yyprintf((stderr, "<%c>", yyc));			\
-  }
-#endif
-#ifndef YY_BEGIN
-#define YY_BEGIN	( ctx->begin= ctx->pos, 1)
-#endif
-#ifndef YY_END
-#define YY_END		( ctx->end= ctx->pos, 1)
-#endif
-#ifdef YY_DEBUG
-# define yyprintf(args)	fprintf args
-#else
-# define yyprintf(args)
-#endif
-#ifndef YYSTYPE
-#define YYSTYPE	int
-#endif
-
-#ifndef YY_PART
-
-typedef struct _yycontext yycontext;
-typedef void (*yyaction)(yycontext *ctx, char *yytext, int yyleng);
-typedef struct _yythunk { int begin, end;  yyaction  action;  struct _yythunk *next; } yythunk;
-
-struct _yycontext {
-  char     *buf;
-  int       buflen;
-  int       pos;
-  int       limit;
-  char     *text;
-  int       textlen;
-  int       begin;
-  int       end;
-  int       textmax;
-  yythunk  *thunks;
-  int       thunkslen;
-  int       thunkpos;
-  YYSTYPE   yy;
-  YYSTYPE  *val;
-  YYSTYPE  *vals;
-  int       valslen;
-#ifdef YY_CTX_MEMBERS
-  YY_CTX_MEMBERS
-#endif
-};
-
-#ifdef YY_CTX_LOCAL
-#define YY_CTX_PARAM_	yycontext *yyctx,
-#define YY_CTX_PARAM	yycontext *yyctx
-#define YY_CTX_ARG_	yyctx,
-#define YY_CTX_ARG	yyctx
-#else
-#define YY_CTX_PARAM_
-#define YY_CTX_PARAM
-#define YY_CTX_ARG_
-#define YY_CTX_ARG
-yycontext yyctx0;
-yycontext *yyctx= &yyctx0;
-#endif
-
-YY_LOCAL(int) yyrefill(yycontext *ctx)
-{
-  int yyn;
-  while (ctx->buflen - ctx->pos < 512)
-    {
-      ctx->buflen *= 2;
-      ctx->buf= (char *)realloc(ctx->buf, ctx->buflen);
-    }
-  YY_INPUT((ctx->buf + ctx->pos), yyn, (ctx->buflen - ctx->pos));
-  if (!yyn) return 0;
-  ctx->limit += yyn;
-  return 1;
-}
-
-YY_LOCAL(int) yymatchDot(yycontext *ctx)
-{
-  if (ctx->pos >= ctx->limit && !yyrefill(ctx)) return 0;
-  ++ctx->pos;
-  return 1;
-}
-
-YY_LOCAL(int) yymatchChar(yycontext *ctx, int c)
-{
-  if (ctx->pos >= ctx->limit && !yyrefill(ctx)) return 0;
-  if ((unsigned char)ctx->buf[ctx->pos] == c)
-    {
-      ++ctx->pos;
-      yyprintf((stderr, "  ok   yymatchChar(ctx, %c) @ %s\n", c, ctx->buf+ctx->pos));
-      return 1;
-    }
-  yyprintf((stderr, "  fail yymatchChar(ctx, %c) @ %s\n", c, ctx->buf+ctx->pos));
-  return 0;
-}
-
-YY_LOCAL(int) yymatchString(yycontext *ctx, char *s)
-{
-  int yysav= ctx->pos;
-  while (*s)
-    {
-      if (ctx->pos >= ctx->limit && !yyrefill(ctx)) return 0;
-      if (ctx->buf[ctx->pos] != *s)
-        {
-          ctx->pos= yysav;
-          return 0;
-        }
-      ++s;
-      ++ctx->pos;
-    }
-  return 1;
-}
-
-YY_LOCAL(int) yymatchClass(yycontext *ctx, unsigned char *bits)
-{
-  int c;
-  if (ctx->pos >= ctx->limit && !yyrefill(ctx)) return 0;
-  c= (unsigned char)ctx->buf[ctx->pos];
-  if (bits[c >> 3] & (1 << (c & 7)))
-    {
-      ++ctx->pos;
-      yyprintf((stderr, "  ok   yymatchClass @ %s\n", ctx->buf+ctx->pos));
-      return 1;
-    }
-  yyprintf((stderr, "  fail yymatchClass @ %s\n", ctx->buf+ctx->pos));
-  return 0;
-}
-
-YY_LOCAL(void) yyDo(yycontext *ctx, yyaction action, int begin, int end)
-{
-  while (ctx->thunkpos >= ctx->thunkslen)
-    {
-      ctx->thunkslen *= 2;
-      ctx->thunks= (yythunk *)realloc(ctx->thunks, sizeof(yythunk) * ctx->thunkslen);
-    }
-  ctx->thunks[ctx->thunkpos].begin=  begin;
-  ctx->thunks[ctx->thunkpos].end=    end;
-  ctx->thunks[ctx->thunkpos].action= action;
-  ++ctx->thunkpos;
-}
-
-YY_LOCAL(int) yyText(yycontext *ctx, int begin, int end)
-{
-  int yyleng= end - begin;
-  if (yyleng <= 0)
-    yyleng= 0;
-  else
-    {
-      while (ctx->textlen < (yyleng + 1))
-	{
-	  ctx->textlen *= 2;
-	  ctx->text= (char *)realloc(ctx->text, ctx->textlen);
-	}
-      memcpy(ctx->text, ctx->buf + begin, yyleng);
-    }
-  ctx->text[yyleng]= '\0';
-  return yyleng;
-}
-
-YY_LOCAL(void) yyDone(yycontext *ctx)
-{
-  int pos;
-  for (pos= 0;  pos < ctx->thunkpos;  ++pos)
-    {
-      yythunk *thunk= &ctx->thunks[pos];
-      int yyleng= thunk->end ? yyText(ctx, thunk->begin, thunk->end) : thunk->begin;
-      yyprintf((stderr, "DO [%d] %p %s\n", pos, thunk->action, ctx->text));
-      thunk->action(ctx, ctx->text, yyleng);
-    }
-  ctx->thunkpos= 0;
-}
-
-YY_LOCAL(void) yyCommit(yycontext *ctx)
-{
-  if ((ctx->limit -= ctx->pos))
-    {
-      memmove(ctx->buf, ctx->buf + ctx->pos, ctx->limit);
-    }
-  ctx->begin -= ctx->pos;
-  ctx->end -= ctx->pos;
-  ctx->pos= ctx->thunkpos= 0;
-}
-
-YY_LOCAL(int) yyAccept(yycontext *ctx, int tp0)
-{
-  if (tp0)
-    {
-      fprintf(stderr, "accept denied at %d\n", tp0);
-      return 0;
-    }
-  else
-    {
-      yyDone(ctx);
-      yyCommit(ctx);
-    }
-  return 1;
-}
-
-YY_LOCAL(void) yyPush(yycontext *ctx, char *text, int count)  { ctx->val += count; }
-YY_LOCAL(void) yyPop(yycontext *ctx, char *text, int count)   { ctx->val -= count; }
-YY_LOCAL(void) yySet(yycontext *ctx, char *text, int count)   { ctx->val[count]= ctx->yy; }
-
-#endif /* YY_PART */
-
-#define	YYACCEPT	yyAccept(ctx, yythunkpos0)
-
-YY_RULE(int) yy_digit(yycontext *ctx); /* 14 */
-YY_RULE(int) yy_float(yycontext *ctx); /* 13 */
-YY_RULE(int) yy_integer(yycontext *ctx); /* 12 */
-YY_RULE(int) yy_rangelet(yycontext *ctx); /* 11 */
-YY_RULE(int) yy_identifier(yycontext *ctx); /* 10 */
-YY_RULE(int) yy_accessors(yycontext *ctx); /* 9 */
-YY_RULE(int) yy_numeric(yycontext *ctx); /* 8 */
-YY_RULE(int) yy_string(yycontext *ctx); /* 7 */
-YY_RULE(int) yy_const(yycontext *ctx); /* 6 */
-YY_RULE(int) yy_hash(yycontext *ctx); /* 5 */
-YY_RULE(int) yy_range(yycontext *ctx); /* 4 */
-YY_RULE(int) yy_entity(yycontext *ctx); /* 3 */
-YY_RULE(int) yy_primary(yycontext *ctx); /* 2 */
-YY_RULE(int) yy_grammar(yycontext *ctx); /* 1 */
-
-YY_ACTION(void) yy_4_const(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_4_const\n"));
-   yy = Qnil; ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_3_const(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_3_const\n"));
-   yy = Qnil; ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_2_const(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_2_const\n"));
-   yy = Qfalse; ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_1_const(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_1_const\n"));
-   yy = Qtrue; ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_1_integer(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_1_integer\n"));
-   yy = rb_funcall(rb_cObject, rb_intern("Integer"), 1, yy_rb_str); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_1_float(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_1_float\n"));
-   yy = rb_funcall(rb_cObject, rb_intern("Float"), 1, yy_rb_str); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_2_string(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_2_string\n"));
-   yy = yy_rb_str; ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_1_string(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_1_string\n"));
-   yy = yy_rb_str; ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_1_range(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_1_range\n"));
-   EMIT("range", Qnil); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_1_rangelet(yycontext *ctx, char *yytext, int yyleng)
-{
-#define var ctx->val[-1]
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_1_rangelet\n"));
-   EMIT("id", var); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-#undef var
-}
-YY_ACTION(void) yy_1_entity(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_1_entity\n"));
-   EMIT("id", yy_rb_str); EMIT("lookup", Qnil); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_5_accessors(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_5_accessors\n"));
-   EMIT("id", yy_rb_str); EMIT("call", Qnil); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_4_accessors(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_4_accessors\n"));
-   EMIT("buildin", rb_str_new2("size")); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_3_accessors(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_3_accessors\n"));
-   EMIT("buildin", rb_str_new2("last")); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_2_accessors(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_2_accessors\n"));
-   EMIT("buildin", rb_str_new2("first")); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_1_accessors(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_1_accessors\n"));
-   EMIT("call", Qnil); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-YY_ACTION(void) yy_3_primary(yycontext *ctx, char *yytext, int yyleng)
-{
-#define var ctx->val[-1]
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_3_primary\n"));
-   EMIT("id", var); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-#undef var
-}
-YY_ACTION(void) yy_2_primary(yycontext *ctx, char *yytext, int yyleng)
-{
-#define var ctx->val[-1]
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_2_primary\n"));
-   EMIT("id", var); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-#undef var
-}
-YY_ACTION(void) yy_1_primary(yycontext *ctx, char *yytext, int yyleng)
-{
-#define var ctx->val[-1]
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_1_primary\n"));
-   EMIT("id", var); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-#undef var
-}
-YY_ACTION(void) yy_1_grammar(yycontext *ctx, char *yytext, int yyleng)
-{
-#define yy ctx->yy
-#define yypos ctx->pos
-#define yythunkpos ctx->thunkpos
-  yyprintf((stderr, "do yy_1_grammar\n"));
-   EMIT("lookup", Qnil); ;
-#undef yythunkpos
-#undef yypos
-#undef yy
-}
-
-YY_RULE(int) yy_digit(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "digit"));  if (!yymatchClass(ctx, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l1;
-  yyprintf((stderr, "  ok   %s @ %s\n", "digit", ctx->buf+ctx->pos));
-  return 1;
-  l1:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "digit", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_float(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "float"));  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_BEGIN)) goto l2;
-  {  int yypos3= ctx->pos, yythunkpos3= ctx->thunkpos;  if (!yymatchChar(ctx, '-')) goto l3;  goto l4;
-  l3:;	  ctx->pos= yypos3; ctx->thunkpos= yythunkpos3;
-  }
-  l4:;	  if (!yy_digit(ctx)) goto l2;  if (!yymatchChar(ctx, '.')) goto l2;  if (!yy_digit(ctx)) goto l2;
-  l5:;	
-  {  int yypos6= ctx->pos, yythunkpos6= ctx->thunkpos;  if (!yy_digit(ctx)) goto l6;  goto l5;
-  l6:;	  ctx->pos= yypos6; ctx->thunkpos= yythunkpos6;
-  }  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_END)) goto l2;  yyDo(ctx, yy_1_float, ctx->begin, ctx->end);
-  yyprintf((stderr, "  ok   %s @ %s\n", "float", ctx->buf+ctx->pos));
-  return 1;
-  l2:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "float", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_integer(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "integer"));  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_BEGIN)) goto l7;
-  {  int yypos8= ctx->pos, yythunkpos8= ctx->thunkpos;  if (!yymatchChar(ctx, '-')) goto l8;  goto l9;
-  l8:;	  ctx->pos= yypos8; ctx->thunkpos= yythunkpos8;
-  }
-  l9:;	  if (!yy_digit(ctx)) goto l7;
-  l10:;	
-  {  int yypos11= ctx->pos, yythunkpos11= ctx->thunkpos;  if (!yy_digit(ctx)) goto l11;  goto l10;
-  l11:;	  ctx->pos= yypos11; ctx->thunkpos= yythunkpos11;
-  }  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_END)) goto l7;  yyDo(ctx, yy_1_integer, ctx->begin, ctx->end);
-  yyprintf((stderr, "  ok   %s @ %s\n", "integer", ctx->buf+ctx->pos));
-  return 1;
-  l7:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "integer", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_rangelet(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;  yyDo(ctx, yyPush, 1, 0);
-  yyprintf((stderr, "%s\n", "rangelet"));
-  {  int yypos13= ctx->pos, yythunkpos13= ctx->thunkpos;  if (!yy_integer(ctx)) goto l14;  yyDo(ctx, yySet, -1, 0);  yyDo(ctx, yy_1_rangelet, ctx->begin, ctx->end);  goto l13;
-  l14:;	  ctx->pos= yypos13; ctx->thunkpos= yythunkpos13;  if (!yy_entity(ctx)) goto l12;
-  }
-  l13:;	
-  yyprintf((stderr, "  ok   %s @ %s\n", "rangelet", ctx->buf+ctx->pos));  yyDo(ctx, yyPop, 1, 0);
-  return 1;
-  l12:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "rangelet", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_identifier(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "identifier"));  if (!yymatchClass(ctx, (unsigned char *)"\000\000\000\000\000\000\000\000\376\377\377\007\376\377\377\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l15;
-  l16:;	
-  {  int yypos17= ctx->pos, yythunkpos17= ctx->thunkpos;  if (!yymatchClass(ctx, (unsigned char *)"\000\000\000\000\000\040\377\003\376\377\377\207\376\377\377\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l17;  goto l16;
-  l17:;	  ctx->pos= yypos17; ctx->thunkpos= yythunkpos17;
-  }
-  {  int yypos18= ctx->pos, yythunkpos18= ctx->thunkpos;  if (!yymatchClass(ctx, (unsigned char *)"\000\000\000\000\002\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l18;  goto l19;
-  l18:;	  ctx->pos= yypos18; ctx->thunkpos= yythunkpos18;
-  }
-  l19:;	
-  yyprintf((stderr, "  ok   %s @ %s\n", "identifier", ctx->buf+ctx->pos));
-  return 1;
-  l15:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "identifier", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_accessors(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "accessors"));
-  {  int yypos21= ctx->pos, yythunkpos21= ctx->thunkpos;  if (!yy_hash(ctx)) goto l22;  yyDo(ctx, yy_1_accessors, ctx->begin, ctx->end);  goto l21;
-  l22:;	  ctx->pos= yypos21; ctx->thunkpos= yythunkpos21;  if (!yymatchString(ctx, ".first")) goto l23;  yyDo(ctx, yy_2_accessors, ctx->begin, ctx->end);  goto l21;
-  l23:;	  ctx->pos= yypos21; ctx->thunkpos= yythunkpos21;  if (!yymatchString(ctx, ".last")) goto l24;  yyDo(ctx, yy_3_accessors, ctx->begin, ctx->end);  goto l21;
-  l24:;	  ctx->pos= yypos21; ctx->thunkpos= yythunkpos21;  if (!yymatchString(ctx, ".size")) goto l25;  yyDo(ctx, yy_4_accessors, ctx->begin, ctx->end);  goto l21;
-  l25:;	  ctx->pos= yypos21; ctx->thunkpos= yythunkpos21;  if (!yymatchChar(ctx, '.')) goto l20;  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_BEGIN)) goto l20;  if (!yy_identifier(ctx)) goto l20;  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_END)) goto l20;  yyDo(ctx, yy_5_accessors, ctx->begin, ctx->end);
-  }
-  l21:;	
-  yyprintf((stderr, "  ok   %s @ %s\n", "accessors", ctx->buf+ctx->pos));
-  return 1;
-  l20:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "accessors", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_numeric(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "numeric"));
-  {  int yypos27= ctx->pos, yythunkpos27= ctx->thunkpos;  if (!yy_float(ctx)) goto l28;  goto l27;
-  l28:;	  ctx->pos= yypos27; ctx->thunkpos= yythunkpos27;  if (!yy_integer(ctx)) goto l26;
-  }
-  l27:;	
-  yyprintf((stderr, "  ok   %s @ %s\n", "numeric", ctx->buf+ctx->pos));
-  return 1;
-  l26:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "numeric", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_string(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "string"));
-  {  int yypos30= ctx->pos, yythunkpos30= ctx->thunkpos;  if (!yymatchClass(ctx, (unsigned char *)"\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l31;  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_BEGIN)) goto l31;
-  l32:;	
-  {  int yypos33= ctx->pos, yythunkpos33= ctx->thunkpos;
-  {  int yypos34= ctx->pos, yythunkpos34= ctx->thunkpos;  if (!yymatchClass(ctx, (unsigned char *)"\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l34;  goto l33;
-  l34:;	  ctx->pos= yypos34; ctx->thunkpos= yythunkpos34;
-  }  if (!yymatchDot(ctx)) goto l33;  goto l32;
-  l33:;	  ctx->pos= yypos33; ctx->thunkpos= yythunkpos33;
-  }  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_END)) goto l31;  if (!yymatchClass(ctx, (unsigned char *)"\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l31;  yyDo(ctx, yy_1_string, ctx->begin, ctx->end);  goto l30;
-  l31:;	  ctx->pos= yypos30; ctx->thunkpos= yythunkpos30;  if (!yymatchClass(ctx, (unsigned char *)"\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l29;  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_BEGIN)) goto l29;
-  l35:;	
-  {  int yypos36= ctx->pos, yythunkpos36= ctx->thunkpos;
-  {  int yypos37= ctx->pos, yythunkpos37= ctx->thunkpos;  if (!yymatchClass(ctx, (unsigned char *)"\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l37;  goto l36;
-  l37:;	  ctx->pos= yypos37; ctx->thunkpos= yythunkpos37;
-  }  if (!yymatchDot(ctx)) goto l36;  goto l35;
-  l36:;	  ctx->pos= yypos36; ctx->thunkpos= yythunkpos36;
-  }  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_END)) goto l29;  if (!yymatchClass(ctx, (unsigned char *)"\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l29;  yyDo(ctx, yy_2_string, ctx->begin, ctx->end);
-  }
-  l30:;	
-  yyprintf((stderr, "  ok   %s @ %s\n", "string", ctx->buf+ctx->pos));
-  return 1;
-  l29:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "string", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_const(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "const"));
-  {  int yypos39= ctx->pos, yythunkpos39= ctx->thunkpos;  if (!yymatchString(ctx, "true")) goto l40;  yyDo(ctx, yy_1_const, ctx->begin, ctx->end);  goto l39;
-  l40:;	  ctx->pos= yypos39; ctx->thunkpos= yythunkpos39;  if (!yymatchString(ctx, "false")) goto l41;  yyDo(ctx, yy_2_const, ctx->begin, ctx->end);  goto l39;
-  l41:;	  ctx->pos= yypos39; ctx->thunkpos= yythunkpos39;  if (!yymatchString(ctx, "nil")) goto l42;  yyDo(ctx, yy_3_const, ctx->begin, ctx->end);  goto l39;
-  l42:;	  ctx->pos= yypos39; ctx->thunkpos= yythunkpos39;  if (!yymatchString(ctx, "null")) goto l38;  yyDo(ctx, yy_4_const, ctx->begin, ctx->end);
-  }
-  l39:;	
-  yyprintf((stderr, "  ok   %s @ %s\n", "const", ctx->buf+ctx->pos));
-  return 1;
-  l38:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "const", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_hash(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "hash"));  if (!yymatchChar(ctx, '[')) goto l43;
-  {  int yypos44= ctx->pos, yythunkpos44= ctx->thunkpos;  if (!yy_primary(ctx)) goto l45;  goto l44;
-  l45:;	  ctx->pos= yypos44; ctx->thunkpos= yythunkpos44;  if (!yy_entity(ctx)) goto l43;
-  }
-  l44:;	  if (!yymatchChar(ctx, ']')) goto l43;
-  yyprintf((stderr, "  ok   %s @ %s\n", "hash", ctx->buf+ctx->pos));
-  return 1;
-  l43:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "hash", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_range(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "range"));  if (!yymatchChar(ctx, '(')) goto l46;  if (!yy_rangelet(ctx)) goto l46;  if (!yymatchString(ctx, "..")) goto l46;  if (!yy_rangelet(ctx)) goto l46;  if (!yymatchChar(ctx, ')')) goto l46;  yyDo(ctx, yy_1_range, ctx->begin, ctx->end);
-  yyprintf((stderr, "  ok   %s @ %s\n", "range", ctx->buf+ctx->pos));
-  return 1;
-  l46:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "range", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_entity(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "entity"));  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_BEGIN)) goto l47;  if (!yy_identifier(ctx)) goto l47;  yyText(ctx, ctx->begin, ctx->end);  if (!(YY_END)) goto l47;  yyDo(ctx, yy_1_entity, ctx->begin, ctx->end);
-  l48:;	
-  {  int yypos49= ctx->pos, yythunkpos49= ctx->thunkpos;  if (!yy_accessors(ctx)) goto l49;  goto l48;
-  l49:;	  ctx->pos= yypos49; ctx->thunkpos= yythunkpos49;
-  }
-  yyprintf((stderr, "  ok   %s @ %s\n", "entity", ctx->buf+ctx->pos));
-  return 1;
-  l47:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "entity", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_primary(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;  yyDo(ctx, yyPush, 1, 0);
-  yyprintf((stderr, "%s\n", "primary"));
-  {  int yypos51= ctx->pos, yythunkpos51= ctx->thunkpos;  if (!yy_const(ctx)) goto l52;  yyDo(ctx, yySet, -1, 0);  yyDo(ctx, yy_1_primary, ctx->begin, ctx->end);  goto l51;
-  l52:;	  ctx->pos= yypos51; ctx->thunkpos= yythunkpos51;  if (!yy_string(ctx)) goto l53;  yyDo(ctx, yySet, -1, 0);  yyDo(ctx, yy_2_primary, ctx->begin, ctx->end);  goto l51;
-  l53:;	  ctx->pos= yypos51; ctx->thunkpos= yythunkpos51;  if (!yy_numeric(ctx)) goto l50;  yyDo(ctx, yySet, -1, 0);  yyDo(ctx, yy_3_primary, ctx->begin, ctx->end);
-  }
-  l51:;	
-  yyprintf((stderr, "  ok   %s @ %s\n", "primary", ctx->buf+ctx->pos));  yyDo(ctx, yyPop, 1, 0);
-  return 1;
-  l50:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "primary", ctx->buf+ctx->pos));
-  return 0;
-}
-YY_RULE(int) yy_grammar(yycontext *ctx)
-{  int yypos0= ctx->pos, yythunkpos0= ctx->thunkpos;
-  yyprintf((stderr, "%s\n", "grammar"));
-  {  int yypos55= ctx->pos, yythunkpos55= ctx->thunkpos;  if (!yy_primary(ctx)) goto l56;  goto l55;
-  l56:;	  ctx->pos= yypos55; ctx->thunkpos= yythunkpos55;  if (!yy_entity(ctx)) goto l57;  goto l55;
-  l57:;	  ctx->pos= yypos55; ctx->thunkpos= yythunkpos55;  if (!yy_range(ctx)) goto l58;  goto l55;
-  l58:;	  ctx->pos= yypos55; ctx->thunkpos= yythunkpos55;  if (!yy_hash(ctx)) goto l54;  yyDo(ctx, yy_1_grammar, ctx->begin, ctx->end);
-  }
-  l55:;	
-  yyprintf((stderr, "  ok   %s @ %s\n", "grammar", ctx->buf+ctx->pos));
-  return 1;
-  l54:;	  ctx->pos= yypos0; ctx->thunkpos= yythunkpos0;
-  yyprintf((stderr, "  fail %s @ %s\n", "grammar", ctx->buf+ctx->pos));
-  return 0;
-}
-
-#ifndef YY_PART
-
-typedef int (*yyrule)(yycontext *ctx);
-
-YY_PARSE(int) YYPARSEFROM(YY_CTX_PARAM_ yyrule yystart)
-{
-  int yyok;
-  if (!yyctx->buflen)
-    {
-      yyctx->buflen= 1024;
-      yyctx->buf= (char *)malloc(yyctx->buflen);
-      yyctx->textlen= 1024;
-      yyctx->text= (char *)malloc(yyctx->textlen);
-      yyctx->thunkslen= 32;
-      yyctx->thunks= (yythunk *)malloc(sizeof(yythunk) * yyctx->thunkslen);
-      yyctx->valslen= 32;
-      yyctx->vals= (YYSTYPE *)malloc(sizeof(YYSTYPE) * yyctx->valslen);
-      yyctx->begin= yyctx->end= yyctx->pos= yyctx->limit= yyctx->thunkpos= 0;
-    }
-  yyctx->begin= yyctx->end= yyctx->pos;
-  yyctx->thunkpos= 0;
-  yyctx->val= yyctx->vals;
-  yyok= yystart(yyctx);
-  if (yyok) yyDone(yyctx);
-  yyCommit(yyctx);
-  return yyok;
-}
-
-YY_PARSE(int) YYPARSE(YY_CTX_PARAM)
-{
-  return YYPARSEFROM(YY_CTX_ARG_ yy_grammar);
-}
-
-#endif
- 
-
-VALUE liquid_context_parse_impl(VALUE self, VALUE text) {
-  char *p; 
-  int len; 
-  yycontext ctx;  
-
-  memset(&ctx, 0, sizeof(yycontext));
-  ctx.p = RSTRING_PTR(text);  
-  ctx.p_len = (int) RSTRING_LEN(text);
-  ctx.rb_tokens = rb_ary_new();
- 
-  yyparse(&ctx);
-
-  return ctx.rb_tokens;
-}

ext/liquid/liquid_context.leg

@@ -1,98 +0,0 @@
-
-%{ 
-//uncomment to get more debug instrumentation
-//
-//#define YY_DEBUG 
-
-#include <ruby.h>
-
-#define EMIT(sym, data) \
-  rb_ary_push(ctx->rb_tokens, rb_ary_new3(2, ID2SYM(rb_intern(sym)), data)); 
-
-#define yy_rb_str rb_str_new(yytext, yyleng)
-
-#define YYSTYPE VALUE
-#define YY_CTX_LOCAL
-#define YY_CTX_MEMBERS VALUE rb_tokens; char *p; int p_len;
-
-#define YY_INPUT(buf, result, max_size) { \
-    result = ctx->p_len;  \
-    if (result>0 || EOF == ctx->p[0]) { \
-      if (max_size < result) { result = max_size; } \
-      strncpy(buf, ctx->p, result); \
-      buf[result] = '\0'; \
-      yyprintf((stderr, "\nREFILLING %d chars now:<%s>", result, buf));      \
-      ctx->p += result; ctx->p_len -= result; \
-      yyprintf((stderr, "\nREFILLING DONE size left: %d <%s>", ctx->p_len, ctx->p));      \
-    } \
-  }
-%}
-
-grammar  = primary
-         | entity
-         | range
-         | hash { EMIT("lookup", Qnil); }
-         ; 
-
-hash     = '[' (primary|entity) ']';
-
-primary  = var:const     { EMIT("id", var); }
-         | var:string    { EMIT("id", var); } 
-         | var:numeric   { EMIT("id", var); } 
-         ;
-
-accessors = hash         { EMIT("call", Qnil); }
-          | '.first'     { EMIT("buildin", rb_str_new2("first")); }
-          | '.last'      { EMIT("buildin", rb_str_new2("last")); }
-          | '.size'      { EMIT("buildin", rb_str_new2("size")); }          
-          | '.' <identifier>   { EMIT("id", yy_rb_str); EMIT("call", Qnil); } 
-          ;
-
-entity  = <identifier>   { EMIT("id", yy_rb_str); EMIT("lookup", Qnil); } 
-          accessors* 
-        ;
-
-rangelet = var:integer   { EMIT("id", var); }
-         | entity 
-         ;
-
-range    = '(' rangelet '..' rangelet ')' { EMIT("range", Qnil); } 
-
-
-string = ['] < ( !['] .  )* > [']  { $$ = yy_rb_str; }
-       | ["] < ( !["] .  )* > ["]  { $$ = yy_rb_str; }
-       ;
-
-numeric = float
-        | integer
-        ;
-
-float = <'-'? digit+'.'digit+>     { $$ = rb_funcall(rb_cObject, rb_intern("Float"), 1, yy_rb_str); }
-integer = <'-'? digit+>           { $$ = rb_funcall(rb_cObject, rb_intern("Integer"), 1, yy_rb_str); }
-
-
-const   = "true"    { $$ = Qtrue; }
-        | 'false'   { $$ = Qfalse; }
-        | 'nil'     { $$ = Qnil; }
-        | 'null'    { $$ = Qnil; }
-        ;
-
-digit = [0-9];
-identifier = [a-zA-Z][a-zA-Z0-9_\-]*[?!]?;
-
-%% 
-
-VALUE liquid_context_parse_impl(VALUE self, VALUE text) {
-  char *p; 
-  int len; 
-  yycontext ctx;  
-
-  memset(&ctx, 0, sizeof(yycontext));
-  ctx.p = RSTRING_PTR(text);  
-  ctx.p_len = (int) RSTRING_LEN(text);
-  ctx.rb_tokens = rb_ary_new();
- 
-  yyparse(&ctx);
-
-  return ctx.rb_tokens;
-}

ext/liquid/liquid_ext.c

@@ -1,13 +0,0 @@
-#include <ruby.h>
-
-static VALUE rb_Liquid;
-static VALUE rb_Parser;
-
-VALUE liquid_context_parse_impl(VALUE text);
-
-void Init_liquid_ext()
-{
-  rb_Liquid = rb_define_module("Liquid");
-  rb_Parser = rb_define_class_under(rb_Liquid, "Parser", rb_cObject);
-  rb_define_singleton_method(rb_Parser, "parse", liquid_context_parse_impl, 1);  
-}

lib/liquid.rb

@@ -20,14 +20,15 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 module Liquid
+  WordRegex                   = RUBY_VERSION < "1.9" ? '\w' : '[[:word:]]'
   FilterSeparator             = /\|/
   ArgumentSeparator           = ','
   FilterArgumentSeparator     = ':'
   VariableAttributeSeparator  = '.'
   TagStart                    = /\{\%/
   TagEnd                      = /\%\}/
-  VariableSignature           = /\(?[\w\-\.\[\]]\)?/
-  VariableSegment             = /[\w\-]/
+  VariableSignature           = /\(?[#{WordRegex}\-\.\[\]]\)?/o
+  VariableSegment             = /[#{WordRegex}\-]/o
   VariableStart               = /\{\{/
   VariableEnd                 = /\}\}/
   VariableIncompleteEnd       = /\}\}?/
@@ -38,7 +39,7 @@ module Liquid
   OtherFilterArgument         = /#{ArgumentSeparator}(?:#{StrictQuotedFragment})/o
   SpacelessFilter             = /^(?:'[^']+'|"[^"]+"|[^'"])*#{FilterSeparator}(?:#{StrictQuotedFragment})(?:#{FirstFilterArgument}(?:#{OtherFilterArgument})*)?/o
   Expression                  = /(?:#{QuotedFragment}(?:#{SpacelessFilter})*)/o
-  TagAttributes               = /(\w+)\s*\:\s*(#{QuotedFragment})/o
+  TagAttributes               = /(#{WordRegex}+)\s*\:\s*(#{QuotedFragment})/o
   AnyStartingTag              = /\{\{|\{\%/
   PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/o
   TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/o
@@ -62,7 +63,6 @@ require 'liquid/standardfilters'
 require 'liquid/condition'
 require 'liquid/module_ex'
 require 'liquid/utils'
-require 'liquid_ext'
 
 # Load all the tags of the standard library
 #

lib/liquid/block.rb

@@ -3,7 +3,7 @@ module Liquid
   class Block < Tag
     IsTag             = /^#{TagStart}/o
     IsVariable        = /^#{VariableStart}/o
-    FullToken         = /^#{TagStart}\s*(\w+)\s*(.*)?#{TagEnd}$/o
+    FullToken         = /^#{TagStart}\s*(#{WordRegex}+)\s*(.*)?#{TagEnd}$/o
     ContentOfVariable = /^#{VariableStart}(.*)#{VariableEnd}$/o
 
     def parse(tokens)
@@ -109,7 +109,7 @@ module Liquid
         end
       end
 
-      output
+      output.join
     end
   end
 end

lib/liquid/context.rb

@@ -25,6 +25,7 @@ module Liquid
       squash_instance_assigns_with_environments
 
       @interrupts = []
+      @variable_cache = {}
     end
 
     def strainer
@@ -40,6 +41,7 @@ module Liquid
 
       filters.each do |f|
         raise ArgumentError, "Expected module but got: #{f.class}" unless f.is_a?(Module)
+        Strainer.add_known_filter(f)
         strainer.extend(f)
       end
     end
@@ -72,26 +74,25 @@ module Liquid
     end
 
     def invoke(method, *args)
-      if strainer.respond_to?(method)
-        strainer.__send__(method, *args)
-      else
-        args.first
-      end
+      strainer.invoke(method, *args)
     end
 
     # Push new local scope on the stack. use <tt>Context#stack</tt> instead
     def push(new_scope={})
+      @variable_cache = {}
       @scopes.unshift(new_scope)
       raise StackLevelError, "Nesting too deep" if @scopes.length > 100
     end
 
     # Merge a hash of variables in the current local scope
     def merge(new_scopes)
+      @variable_cache = {}
       @scopes[0].merge!(new_scopes)
     end
 
     # Pop from the stack. use <tt>Context#stack</tt> instead
     def pop
+      @variable_cache = {}
       raise ContextError if @scopes.size == 1
       @scopes.shift
     end
@@ -105,13 +106,16 @@ module Liquid
     #
     #   context['var]  #=> nil
     def stack(new_scope={})
+      @variable_cache = {}
       push(new_scope)
       yield
     ensure
       pop
+      @variable_cache = {}
     end
 
     def clear_instance_assigns
+      @variable_cache = {}
       @scopes[0] = {}
     end
 
@@ -174,6 +178,7 @@ module Liquid
 
     # Only allow String, Numeric, Hash, Array, Proc, Boolean or <tt>Liquid::Drop</tt>
     def []=(key, value)
+      @variable_cache[key] = value
       @scopes[0][key] = value
     end
 
@@ -203,6 +208,10 @@ module Liquid
 
       # Fetches an object starting at the local scope and then moving up the hierachy
       def find_variable(key)
+        if val = @variable_cache[key]
+          return val
+        end
+
         scope = @scopes.find { |s| s.has_key?(key) }
 
         if scope.nil?
@@ -216,6 +225,7 @@ module Liquid
 
         scope     ||= @environments.last || @scopes.last
         variable  ||= lookup_and_evaluate(scope, key)
+        @variable_cache[key] = variable
 
         return variable
       end

lib/liquid/drop.rb

@@ -1,3 +1,5 @@
+require 'set'
+
 module Liquid
 
   # A drop in liquid is a class which allows you to export DOM like things to liquid.
@@ -31,8 +33,8 @@ module Liquid
 
     # called by liquid to invoke a drop
     def invoke_drop(method_or_key)
-      if method_or_key && method_or_key != EMPTY_STRING && self.class.public_method_defined?(method_or_key.to_s.to_sym)
-        send(method_or_key.to_s.to_sym)
+      if method_or_key && method_or_key != EMPTY_STRING && self.class.invokable?(method_or_key)
+        send(method_or_key)
       else
         before_method(method_or_key)
       end
@@ -47,5 +49,13 @@ module Liquid
     end
 
     alias :[] :invoke_drop
+
+    private
+
+    # Check for method existence without invoking respond_to?, which creates symbols
+    def self.invokable?(method_name)
+      @invokable_methods ||= Set.new((public_instance_methods - Liquid::Drop.public_instance_methods).map(&:to_s))
+      @invokable_methods.include?(method_name.to_s)
+    end
   end
 end

lib/liquid/htmltags.rb

@@ -1,13 +1,11 @@
 module Liquid
   class TableRow < Block
-    Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)/o
+    Syntax = /(#{WordRegex}+)\s+in\s+(#{QuotedFragment}+)/o
 
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax
         @variable_name = $1
         @collection_name = $2
-        @idx_i = "#{$1}-#{$2}-i"
-        @idx_col = "#{$1}-#{$2}-c"
         @attributes = {}
         markup.scan(TagAttributes) do |key, value|
           @attributes[key] = value
@@ -20,8 +18,6 @@ module Liquid
     end
 
     def render(context)
-      context.registers[:tablerowloop] ||= Hash.new(0)
-
       collection = context[@collection_name] or return ''
 
       from = @attributes['offset'] ? context[@attributes['offset']].to_i : 0
@@ -36,15 +32,26 @@ module Liquid
       row = 1
       col = 0
 
-      result = ["<tr class=\"row1\">\n"]
+      result = "<tr class=\"row1\">\n"
       context.stack do
 
-        context.registers[:tablerowloop][@idx]
-        context['tablerowloop'] = lambda { Tablerowloop.new(@idx_i, @idx_col, length) }
         collection.each_with_index do |item, index|
-          context.registers[:tablerowloop][@idx_i] = index
-          context.registers[:tablerowloop][@idx_col] = col
-          context[@variable_name] = item          
+          context[@variable_name] = item
+          context['tablerowloop'] = {
+            'length'  => length,
+            'index'   => index + 1,
+            'index0'  => index,
+            'col'     => col + 1,
+            'col0'    => col,
+            'index0'  => index,
+            'rindex'  => length - index,
+            'rindex0' => length - index - 1,
+            'first'   => (index == 0),
+            'last'    => (index == length - 1),
+            'col_first' => (col == 0),
+            'col_last'  => (col == cols - 1)
+          }
+
 
           col += 1
 
@@ -61,58 +68,6 @@ module Liquid
       result << "</tr>\n"
       result
     end
-
-
-
-    private
-
-      class Tablerowloop < Liquid::Drop
-        attr_accessor :length
-
-        def initialize(idx_i, idx_col, length)
-          @idx_i, @idx_col, @length = idx_i, idx_col, length
-        end
-
-        def index
-          @context.registers[:tablerowloop][@idx_i] + 1
-        end
-
-        def index0
-          @context.registers[:tablerowloop][@idx_i]
-        end
-
-        def rindex
-          length - @context.registers[:tablerowloop][@idx_i]
-        end
-
-        def rindex0
-          length - @context.registers[:tablerowloop][@idx_i] - 1
-        end
-
-        def first
-          (@context.registers[:tablerowloop][@idx_i] == 0)
-        end
-
-        def last
-          (@context.registers[:tablerowloop][@idx_i] == length - 1)
-        end
-
-        def col
-          @context.registers[:tablerowloop][@idx_col] + 1
-        end
-
-        def col0
-          @context.registers[:tablerowloop][@idx_col]
-        end
-
-        def col_first
-          (@context.registers[:tablerowloop][@idx_col] == 0)
-        end
-
-        def col_last
-          (@context.registers[:tablerowloop][@idx_col] == cols - 1)
-        end
-      end
   end
 
   Template.register_tag('tablerow', TableRow)

lib/liquid/strainer.rb

@@ -2,24 +2,15 @@ require 'set'
 
 module Liquid
 
-  parent_object = if defined? BlankObject
-    BlankObject
-  else
-    Object
-  end
-
   # Strainer is the parent class for the filters system.
-  # New filters are mixed into the strainer class which is then instanciated for each liquid template render run.
+  # New filters are mixed into the strainer class which is then instantiated for each liquid template render run.
   #
-  # One of the strainer's responsibilities is to keep malicious method calls out
-  class Strainer < parent_object #:nodoc:
-    INTERNAL_METHOD = /^__/
-    @@required_methods = Set.new([:__id__, :__send__, :respond_to?, :kind_of?, :extend, :methods, :singleton_methods, :class, :object_id])
-
-    # Ruby 1.9.2 introduces Object#respond_to_missing?, which is invoked by Object#respond_to?
-    @@required_methods << :respond_to_missing? if Object.respond_to? :respond_to_missing?
-
+  # The Strainer only allows method calls defined in filters given to it via Strainer.global_filter,
+  # Context#add_filters or Template.register_filter
+  class Strainer #:nodoc:
     @@filters = {}
+    @@known_filters = Set.new
+    @@known_methods = Set.new
 
     def initialize(context)
       @context = context
@@ -27,28 +18,36 @@ module Liquid
 
     def self.global_filter(filter)
       raise ArgumentError, "Passed filter is not a module" unless filter.is_a?(Module)
+      add_known_filter(filter)
       @@filters[filter.name] = filter
     end
 
+    def self.add_known_filter(filter)
+      unless @@known_filters.include?(filter)
+        @@method_blacklist ||= Set.new(Strainer.instance_methods.map(&:to_s))
+        new_methods = filter.instance_methods.map(&:to_s)
+        new_methods.reject!{ |m| @@method_blacklist.include?(m) }
+        @@known_methods.merge(new_methods)
+        @@known_filters.add(filter)
+      end
+    end
+
     def self.create(context)
       strainer = Strainer.new(context)
       @@filters.each { |k,m| strainer.extend(m) }
       strainer
     end
 
-    def respond_to?(method, include_private = false)
-      method_name = method.to_s
-      return false if method_name =~ INTERNAL_METHOD
-      return false if @@required_methods.include?(method_name)
-      super
+    def invoke(method, *args)
+      if invokable?(method)
+        send(method, *args)
+      else
+        args.first
+      end
     end
 
-    # remove all standard methods from the bucket so circumvent security
-    # problems
-    instance_methods.each do |m|
-      unless @@required_methods.include?(m.to_sym)
-        undef_method m
-      end
+    def invokable?(method)
+      @@known_methods.include?(method.to_s) && respond_to?(method)
     end
   end
 end

lib/liquid/tags/capture.rb

@@ -12,7 +12,7 @@ module Liquid
   # in a sidebar or footer.
   #
   class Capture < Block
-    Syntax = /(\w+)/
+    Syntax = /(#{WordRegex}+)/o
 
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax

lib/liquid/tags/case.rb

@@ -31,7 +31,7 @@ module Liquid
       context.stack do          
         execute_else_block = true
         
-        output = []
+        output = ''
         @blocks.each do |block|
           if block.else? 
             return render_all(block.attachment, context) if execute_else_block

lib/liquid/tags/for.rb

@@ -44,14 +44,13 @@ module Liquid
   # forloop.last:: Returns true if the item is the last item.
   #
   class For < Block                                             
-    Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o
+    Syntax = /(#{WordRegex}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/ou
   
     def initialize(tag_name, markup, tokens)
       if markup =~ Syntax
         @variable_name = $1
         @collection_name = $2
-        @name = "#{$1}-#{$2}"         
-        @idx = "#{@name}-i"
+        @name = "#{$1}-#{$2}"           
         @reversed = $3             
         @attributes = {}
         markup.scan(TagAttributes) do |key, value|
@@ -88,13 +87,14 @@ module Liquid
       limit = context[@attributes['limit']]
       to    = limit ? limit.to_i + from : nil  
 
+
       segment = Utils.slice_collection_using_each(collection, from, to)
 
       return render_else(context) if segment.empty?
       
       segment.reverse! if @reversed
 
-      result = []
+      result = ''
         
       length = segment.length            
             
@@ -102,10 +102,17 @@ module Liquid
       context.registers[:for][@name] = from + segment.length
               
       context.stack do
-        context['forloop'] = lambda { Forloop.new(@name, @idx, length) }
         segment.each_with_index do |item, index|
-          context.registers[:for][@idx] = index
           context[@variable_name] = item
+          context['forloop'] = {
+            'name'    => @name,
+            'length'  => length,
+            'index'   => index + 1, 
+            'index0'  => index, 
+            'rindex'  => length - index,
+            'rindex0' => length - index - 1,
+            'first'   => (index == 0),
+            'last'    => (index == length - 1) }
 
           result << render_all(@for_block, context)
 
@@ -122,39 +129,6 @@ module Liquid
 
     private
 
-      class Forloop < Liquid::Drop
-        attr_accessor :name, :length
-
-        def initialize(name, idx, length)
-          @name, @idx, @length = name, idx, length
-        end
-
-        def index
-          @context.registers[:for][@idx] + 1
-        end
-
-        def index0
-          @context.registers[:for][@idx]
-        end
-
-        def rindex
-          length - @context.registers[:for][@idx]
-        end
-
-        def rindex0
-          length - @context.registers[:for][@idx] - 1
-        end
-
-        def first
-          (@context.registers[:for][@idx] == 0)
-        end
-
-        def last
-          (@context.registers[:for][@idx] == length - 1)
-        end
-      end
-
-
       def render_else(context)
         return @else_block ? [render_all(@else_block, context)] : ''
       end

lib/liquid/tags/include.rb

@@ -2,10 +2,10 @@ module Liquid
   class Include < Tag
     Syntax = /(#{QuotedFragment}+)(\s+(?:with|for)\s+(#{QuotedFragment}+))?/o
   
-    def initialize(tag_name, markup, tokens)      
+    def initialize(tag_name, markup, tokens)
       if markup =~ Syntax
 
-        @template_name = $1        
+        @template_name = $1
         @variable_name = $3
         @attributes    = {}
 
@@ -24,8 +24,7 @@ module Liquid
     end
     
     def render(context)
-      source = _read_template_from_file_system(context)
-      partial = Liquid::Template.parse(source)
+      partial = load_cached_partial(context)
       variable = context[@variable_name || @template_name[1..-2]]
       
       context.stack do
@@ -46,7 +45,21 @@ module Liquid
     end
    
     private
-      def _read_template_from_file_system(context)
+      def load_cached_partial(context)
+        cached_partials = context.registers[:cached_partials] || {}
+        template_name = context[@template_name]
+
+        if cached = cached_partials[template_name]
+          return cached
+        end
+        source = read_template_from_file_system(context)
+        partial = Liquid::Template.parse(source)
+        cached_partials[template_name] = partial
+        context.registers[:cached_partials] = cached_partials
+        partial
+      end
+
+      def read_template_from_file_system(context)
         file_system = context.registers[:file_system] || Liquid::Template.file_system
       
         # make read_template_file call backwards-compatible.

lib/liquid/template.rb

@@ -121,7 +121,8 @@ module Liquid
       begin
         # render the nodelist.
         # for performance reasons we get a array back here. join will make a string out of it
-        @root.render(context).join        
+        result = @root.render(context)
+        result.respond_to?(:join) ? result.join : result
       ensure
         @errors = context.errors
       end

lib/liquid/variable.rb

@@ -11,7 +11,7 @@ module Liquid
   #   {{ user | link }}
   #
   class Variable
-    FilterParser = /(?:#{FilterSeparator}|(?:\s*(?!(?:#{FilterSeparator}))(?:#{QuotedFragment}|\S+)\s*)+)/o
+    FilterParser = /(?:#{FilterSeparator}|(?:\s*(?:#{QuotedFragment}|#{ArgumentSeparator})\s*)+)/o
     attr_accessor :filters, :name
 
     def initialize(markup)
@@ -23,10 +23,10 @@ module Liquid
         if match[2].match(/#{FilterSeparator}\s*(.*)/o)
           filters = Regexp.last_match(1).scan(FilterParser)
           filters.each do |f|
-            if matches = f.match(/\s*(\w+)/)
+            if matches = f.match(/\s*(#{WordRegex}+)(?:\s*#{FilterArgumentSeparator}(.*))?/)
               filtername = matches[1]
-              filterargs = f.scan(/(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})\s*(#{QuotedFragment})/o).flatten
-              @filters << [filtername.to_sym, filterargs]
+              filterargs = matches[2].to_s.scan(/(?:\A|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o).flatten
+              @filters << [filtername, filterargs]
             end
           end
         end
@@ -36,9 +36,16 @@ module Liquid
     def render(context)
       return '' if @name.nil?
       @filters.inject(context[@name]) do |output, filter|
-        filterargs = filter[1].to_a.collect do |a|
-          context[a]
+        filterargs = []
+        keyword_args = {}
+        filter[1].to_a.each do |a|
+          if matches = a.match(/\A#{TagAttributes}\z/o)
+            keyword_args[matches[1]] = context[matches[2]]
+          else
+            filterargs << context[a]
+          end
         end
+        filterargs << keyword_args unless keyword_args.empty?
         begin
           output = context.invoke(filter[0], output, *filterargs)
         rescue FilterNotFound

liquid.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = "liquid"
-  s.version     = "2.4.1"
+  s.version     = "2.5.0"
   s.platform    = Gem::Platform::RUBY
   s.summary     = "A secure, non-evaling end user template engine with aesthetic markup."
   s.authors     = ["Tobias Luetke"]

performance/benchmark.rb

@@ -4,8 +4,8 @@ require File.dirname(__FILE__) + '/theme_runner'
 
 profiler = ThemeRunner.new
 
-Benchmark.bm do |x|
-#  x.report("parse:")   { 100.times { profiler.compile } }
+Benchmark.bmbm do |x|
+  x.report("parse:")   { 100.times { profiler.compile } }
   x.report("parse & run:")   { 100.times { profiler.run } }
 end
 

performance/profile.rb

@@ -6,12 +6,14 @@ profiler = ThemeRunner.new
 
 puts 'Running profiler...'
 
-results  = profiler.run_profile
+results  = profiler.run
 
 puts 'Success'
+puts
 
-filename = (ENV['TMP'] || '/tmp') + "/callgrind.liquid.txt"
-File.open(filename, "w+") do |fp| 
-  RubyProf::CallTreePrinter.new(results).print(fp, :print_file => true) 
+[RubyProf::FlatPrinter, RubyProf::GraphPrinter, RubyProf::GraphHtmlPrinter, RubyProf::CallTreePrinter].each do |klass|
+  filename = (ENV['TMP'] || '/tmp') + (klass.name.include?('Html') ? "/liquid.#{klass.name.downcase}.html" : "/callgrind.liquid.#{klass.name.downcase}.txt")
+  filename.gsub!(/:+/, '_')
+  File.open(filename, "w+") { |fp| klass.new(results).print(fp, :print_file => true) }
+  $stderr.puts "wrote #{klass.name} output to #{filename}"
 end
-$stderr.puts "wrote RubyProf::CallTreePrinter output to #{filename}"

performance/theme_runner.rb

@@ -14,6 +14,17 @@ require File.dirname(__FILE__) + '/shopify/liquid'
 require File.dirname(__FILE__) + '/shopify/database.rb'
 
 class ThemeRunner
+  class FileSystem
+
+    def initialize(path)
+      @path = path
+    end
+
+    # Called by Liquid to retrieve a template file
+    def read_template_file(template_path, context)
+      File.read(@path + '/' + template_path + '.liquid')
+    end
+  end
 
   # Load all templates into memory, do this now so that 
   # we don't profile IO. 
@@ -31,6 +42,7 @@ class ThemeRunner
     # Dup assigns because will make some changes to them
 
     @tests.each do |liquid, layout, template_name|
+
       tmpl = Liquid::Template.new
       tmpl.parse(liquid)
       tmpl = Liquid::Template.new
@@ -46,14 +58,14 @@ class ThemeRunner
 
       # Compute page_tempalte outside of profiler run, uninteresting to profiler
       page_template = File.basename(template_name, File.extname(template_name))
-      compile_and_render(liquid, layout, assigns, page_template) 
+      compile_and_render(liquid, layout, assigns, page_template, template_name)
        
     end
   end
 
 
   def run_profile
-    RubyProf.measure_mode = RubyProf::PROCESS_TIME
+    RubyProf.measure_mode = RubyProf::WALL_TIME
 
     # Dup assigns because will make some changes to them
     assigns = Database.tables.dup
@@ -73,7 +85,7 @@ class ThemeRunner
       html = nil
 
       RubyProf.resume
-      html = compile_and_render(liquid, layout, assigns, page_template) 
+      html = compile_and_render(liquid, layout, assigns, page_template, template_name)
       RubyProf.pause
       
 
@@ -87,10 +99,11 @@ class ThemeRunner
     RubyProf.stop
   end
 
-  def compile_and_render(template, layout, assigns, page_template)    
+  def compile_and_render(template, layout, assigns, page_template, template_file)
     tmpl = Liquid::Template.new
     tmpl.assigns['page_title'] = 'Page title'
     tmpl.assigns['template'] = page_template
+    tmpl.registers[:file_system] = ThemeRunner::FileSystem.new(File.dirname(template_file))
 
     content_for_layout = tmpl.parse(template).render(assigns)
 

test/liquid/assign_test.rb

@@ -12,7 +12,13 @@ class AssignTest < Test::Unit::TestCase
                            '{% assign foo = values %}.{{ foo[1] }}.',
                            'values' => %w{foo bar baz})
   end
-  
+
+  def test_assigned_utf8_variable
+    assert_template_result('.bar.',
+                           "{% assign foo\u6000 = values %}.{{ foo\u6000[1] }}.",
+                           'values' => %w{foo bar baz})
+  end
+
   def test_assign_with_filter
     assert_template_result('.bar.',
                            '{% assign foo = values | split: "," %}.{{ foo[1] }}.',

test/liquid/block_test.rb

@@ -51,6 +51,14 @@ class BlockTest < Test::Unit::TestCase
     end
   end
 
+  def test_with_custom_utf8_tag
+    Liquid::Template.register_tag("testtag\u6000", Block)
+
+    assert_nothing_thrown do
+      template = Liquid::Template.parse( "{% testtag\u6000 something\u6000 %} {% endtesttag\u6000 %}")
+    end
+  end
+
   private
     def block_types(nodelist)
       nodelist.collect { |node| node.class }

test/liquid/capture_test.rb

@@ -7,6 +7,10 @@ class CaptureTest < Test::Unit::TestCase
     assert_template_result("test string", "{% capture 'var' %}test string{% endcapture %}{{var}}", {})
   end
 
+  def test_captures_block_content_in_utf8_variable
+    assert_template_result("test string", "{% capture var\u6000 %}test string{% endcapture %}{{var\u6000}}", {})
+  end
+
   def test_capture_to_variable_from_outer_scope_if_existing
     template_source = <<-END_TEMPLATE
     {% assign var = '' %}

test/liquid/context_test.rb

@@ -98,6 +98,11 @@ class ContextTest < Test::Unit::TestCase
     assert_equal nil, @context['nil']
   end
 
+  def test_utf8_variables
+    @context["chinese\u6000variable"] = 'chinese'
+    assert_equal 'chinese', @context["chinese\u6000variable"]
+  end
+
   def test_variables_not_existing
     assert_equal nil, @context['does_not_exist']
   end
@@ -189,10 +194,10 @@ class ContextTest < Test::Unit::TestCase
     end
 
     context = Context.new
-    methods_before = context.strainer.methods.map { |method| method.to_s }
+    assert_equal "Wookie", context.invoke("hi", "Wookie")
+
     context.add_filters(filter)
-    methods_after = context.strainer.methods.map { |method| method.to_s }
-    assert_equal (methods_before + ["hi"]).sort, methods_after.sort
+    assert_equal "Wookie hi!", context.invoke("hi", "Wookie")
   end
 
   def test_add_item_in_outer_scope
@@ -254,16 +259,12 @@ class ContextTest < Test::Unit::TestCase
     @context['test'] = {'test' => [1,2,3,4,5]}
 
     assert_equal 1, @context['test.test[0]']
-  end
 
-  def test_recoursive_array_notation_for_hash
     @context['test'] = [{'test' => 'worked'}]
 
     assert_equal 'worked', @context['test[0].test']
   end
 
-
-
   def test_hash_to_array_transition
     @context['colors'] = {
       'Blue'    => ['003366','336699', '6699CC', '99CCFF'],
@@ -319,7 +320,7 @@ class ContextTest < Test::Unit::TestCase
     @context['nested'] = {'var' => 'tags'}
     @context['products'] = {'count' => 5, 'tags' => ['deepsnow', 'freestyle'] }
 
-    #assert_equal 'deepsnow', @context['products[var].first']
+    assert_equal 'deepsnow', @context['products[var].first']
     assert_equal 'freestyle', @context['products[nested.var].last']
   end
 

test/liquid/drop_test.rb

@@ -115,6 +115,13 @@ class DropsTest < Test::Unit::TestCase
     assert_equal '  ', output
   end
 
+  def test_object_methods_not_allowed
+    [:dup, :clone, :singleton_class, :eval, :class_eval, :inspect].each do |method|
+      output = Liquid::Template.parse(" {{ product.#{method} }} ").render('product' => ProductDrop.new)
+      assert_equal '  ', output
+    end
+  end
+
   def test_scope
     assert_equal '1', Liquid::Template.parse( '{{ context.scopes }}'  ).render('context' => ContextDrop.new)
     assert_equal '2', Liquid::Template.parse( '{%for i in dummy%}{{ context.scopes }}{%endfor%}'  ).render('context' => ContextDrop.new, 'dummy' => [1])

test/liquid/filter_test.rb

@@ -16,6 +16,12 @@ module CanadianMoneyFilter
   end
 end
 
+module SubstituteFilter
+  def substitute(input, params={})
+    input.gsub(/%\{(\w+)\}/) { |match| params[$1] }
+  end
+end
+
 class FiltersTest < Test::Unit::TestCase
   include Liquid
 
@@ -92,6 +98,13 @@ class FiltersTest < Test::Unit::TestCase
 
     assert_equal 1000, Variable.new("var | xyzzy").render(@context)
   end
+
+  def test_filter_with_keyword_arguments
+    @context['surname'] = 'john'
+    @context.add_filters(SubstituteFilter)
+    output = Variable.new(%! 'hello %{first_name}, %{last_name}' | substitute: first_name: surname, last_name: 'doe' !).render(@context)
+    assert_equal 'hello john, doe', output
+  end
 end
 
 class FiltersInTemplate < Test::Unit::TestCase

test/liquid/parser_test.rb

@@ -1,93 +0,0 @@
-require 'test_helper'
-
-
-class ParserTest < Test::Unit::TestCase
-  include Liquid
-
-
-  def test_strings
-    assert_equal [[:id, "string"]], Parser.parse('"string"')
-    assert_equal [[:id, "string"]], Parser.parse('\'string\'')
-  end
-
-  def test_integer
-    assert_equal [[:id, 1]], Parser.parse('1')
-    assert_equal [[:id, 100001]], Parser.parse('100001')
-  end
-
-  def test_float
-    assert_equal [[:id, 1.1]], Parser.parse('1.1')
-    assert_equal [[:id, 1.55435]], Parser.parse('1.55435')
-  end
-
-  def test_null
-    assert_equal [[:id, nil]], Parser.parse('null')
-    assert_equal [[:id, nil]], Parser.parse('nil')
-  end
-
-  def test_bool
-    assert_equal [[:id, true]], Parser.parse('true')
-    assert_equal [[:id, false]], Parser.parse('false')
-  end
-
-  def test_ranges
-    assert_equal [[:id, 1], [:id, 5], [:range, nil]], Parser.parse('(1..5)')
-    assert_equal [[:id, 100], [:id, 500], [:range, nil]], Parser.parse('(100..500)')    
-  end
-
-  def test_ranges_with_lookups
-    assert_equal [[:id, 1], [:id, "test"], [:lookup, nil], [:range, nil]], Parser.parse('(1..test)')
-  end
-
-  def test_lookups
-    assert_equal [[:id, "variable"], [:lookup, nil]], Parser.parse('variable')
-    assert_equal [[:id, "underscored_variable"], [:lookup, nil]], Parser.parse('underscored_variable')
-    assert_equal [[:id, "c"], [:lookup, nil]], Parser.parse('c')
-  end
-
-  def test_global_hash
-    assert_equal [[:id, true], [:lookup, nil]], Parser.parse('[true]')
-
-    assert_equal [[:id, "string"], [:lookup, nil]], Parser.parse('["string"]')
-    assert_equal [[:id, 5.55], [:lookup, nil]], Parser.parse('[5.55]')
-    assert_equal [[:id, 0], [:lookup, nil]], Parser.parse('[0]')
-    assert_equal [[:id, "variable"], [:lookup, nil], [:lookup, nil]], Parser.parse('[variable]')
-  end
-
-  def test_descent
-    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "variable2"], [:call, nil]], Parser.parse('variable1.variable2')
-    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "variable2"], [:call, nil], [:id, "variable3"], [:call, nil]], Parser.parse('variable1.variable2.variable3')
-    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "under_score"], [:call, nil]], Parser.parse('variable1.under_score')
-    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "question?"], [:call, nil]], Parser.parse('variable1.question?')    
-    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "exclaimation!"], [:call, nil]], Parser.parse('variable1.exclaimation!')    
-  end
-
-  def test_descent_hash
-    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "variable2"], [:call, nil]], Parser.parse('variable1["variable2"]')  
-    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "variable2"], [:lookup, nil], [:call, nil]], Parser.parse('variable1[variable2]')
-  end
-
-  def test_buildin 
-    assert_equal [[:id, "first"], [:lookup, nil]], Parser.parse('first')  
-
-    assert_equal [[:id, "var"], [:lookup, nil], [:buildin, "first"]], Parser.parse('var.first')
-    assert_equal [[:id, "var"], [:lookup, nil], [:buildin, "last"]], Parser.parse('var.last')
-    assert_equal [[:id, "var"], [:lookup, nil], [:buildin, "size"]], Parser.parse('var.size')
-
-  end
-
-  def test_descent_hash_descent
-    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "test1"], [:lookup, nil], [:id, "test2"], [:call, nil], [:call, nil]], 
-      Parser.parse('variable1[test1.test2]'), "resolove: variable1[test1.test2]"
-
-    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "test1"], [:lookup, nil], [:id, "test2"], [:call, nil], [:call, nil]], 
-      Parser.parse('variable1[test1["test2"]]'), 'resolove: variable1[test1["test2"]]'
-
-    assert_equal [[:id, "variable1"], [:lookup, nil], [:id, "test1"], [:lookup, nil], [:id, "test2"], [:lookup, nil], [:call, nil], [:call, nil]], 
-      Parser.parse('variable1[test1[test2]]'), "resolove: variable1[test1[test2]]"
-
-  end
-
-
-
-end

test/liquid/parsing_quirks_test.rb

@@ -39,7 +39,7 @@ class ParsingQuirksTest < Test::Unit::TestCase
 
   def test_meaningless_parens
     assigns = {'b' => 'bar', 'c' => 'baz'}
-    markup = "a == 'foo' or b == 'bar' and c == 'baz' or false"
+    markup = "a == 'foo' or (b == 'bar' and c == 'baz') or false"
     assert_template_result(' YES ',"{% if #{markup} %} YES {% endif %}", assigns)
   end
 

test/liquid/security_test.rb

@@ -38,4 +38,27 @@ class SecurityTest < Test::Unit::TestCase
 
     assert_equal expected, Template.parse(text).render(@assigns, :filters => SecurityFilter)
   end
+
+  def test_does_not_add_filters_to_symbol_table
+    current_symbols = Symbol.all_symbols
+
+    test = %( {{ "some_string" | a_bad_filter }} )
+
+    template = Template.parse(test)
+    assert_equal [], (Symbol.all_symbols - current_symbols)
+
+    template.render
+    assert_equal [], (Symbol.all_symbols - current_symbols)
+  end
+
+  def test_does_not_add_drop_methods_to_symbol_table
+    current_symbols = Symbol.all_symbols
+
+    drop = Drop.new
+    drop.invoke_drop("custom_method_1")
+    drop.invoke_drop("custom_method_2")
+    drop.invoke_drop("custom_method_3")
+
+    assert_equal [], (Symbol.all_symbols - current_symbols)
+  end
 end # SecurityTest

test/liquid/strainer_test.rb

@@ -3,23 +3,50 @@ require 'test_helper'
 class StrainerTest < Test::Unit::TestCase
   include Liquid
 
+  module AccessScopeFilters
+    def public_filter
+      "public"
+    end
+
+    def private_filter
+      "private"
+    end
+    private :private_filter
+  end
+
+  Strainer.global_filter(AccessScopeFilters)
+
   def test_strainer
     strainer = Strainer.create(nil)
-    assert_equal false, strainer.respond_to?('__test__')
-    assert_equal false, strainer.respond_to?('test')
-    assert_equal false, strainer.respond_to?('instance_eval')
-    assert_equal false, strainer.respond_to?('__send__')
-    assert_equal true, strainer.respond_to?('size') # from the standard lib
+    assert_equal 5, strainer.invoke('size', 'input')
+    assert_equal "public", strainer.invoke("public_filter")
   end
 
-  def test_should_respond_to_two_parameters
+  def test_strainer_only_invokes_public_filter_methods
     strainer = Strainer.create(nil)
-    assert_equal true, strainer.respond_to?('size', false)
+    assert_equal false, strainer.invokable?('__test__')
+    assert_equal false, strainer.invokable?('test')
+    assert_equal false, strainer.invokable?('instance_eval')
+    assert_equal false, strainer.invokable?('__send__')
+    assert_equal true, strainer.invokable?('size') # from the standard lib
   end
 
-  # Asserts that Object#respond_to_missing? is not being undefined in Ruby versions where it has been implemented
-  # Currently this method is only present in Ruby v1.9.2, or higher
-  def test_object_respond_to_missing
-    assert_equal Object.respond_to?(:respond_to_missing?), Strainer.create(nil).respond_to?(:respond_to_missing?)
+  def test_strainer_returns_nil_if_no_filter_method_found
+    strainer = Strainer.create(nil)
+    assert_nil strainer.invoke("private_filter")
+    assert_nil strainer.invoke("undef_the_filter")
   end
+
+  def test_strainer_returns_first_argument_if_no_method_and_arguments_given
+    strainer = Strainer.create(nil)
+    assert_equal "password", strainer.invoke("undef_the_method", "password")
+  end
+
+  def test_strainer_only_allows_methods_defined_in_filters
+    strainer = Strainer.create(nil)
+    assert_equal "1 + 1", strainer.invoke("instance_eval", "1 + 1")
+    assert_equal "puts",  strainer.invoke("__send__", "puts", "Hi Mom")
+    assert_equal "has_method?", strainer.invoke("invoke", "has_method?", "invoke")
+  end
+
 end # StrainerTest

test/liquid/tags/for_tag_test.rb

@@ -25,6 +25,11 @@ HERE
     assert_template_result(expected,template,'array' => [1,2,3])
   end
 
+  def test_utf8_for
+    assigns = {"array\u6000chinese" => [1,2,3]}
+    assert_template_result('123', "{% for item\u6000chinese in array\u6000chinese %}{{ item\u6000chinese }}{% endfor %}", assigns)
+  end
+
   def test_for_reversed
     assigns = {'array' => [ 1, 2, 3] }
     assert_template_result('321','{%for item in array reversed %}{{item}}{%endfor%}',assigns)

test/liquid/tags/html_tag_test.rb

@@ -26,6 +26,12 @@ class HtmlTagTest < Test::Unit::TestCase
                            'numbers' => [])
   end
 
+  def test_utf8_html_table
+    assert_template_result("<tr class=\"row1\">\n<td class=\"col1\"> 1 </td></tr>\n",
+                           "{% tablerow n\u6000 in numbers\u6000 %} {{n\u6000}} {% endtablerow %}",
+                           "numbers\u6000" => [1])
+  end
+
   def test_html_table_with_different_cols
     assert_template_result("<tr class=\"row1\">\n<td class=\"col1\"> 1 </td><td class=\"col2\"> 2 </td><td class=\"col3\"> 3 </td><td class=\"col4\"> 4 </td><td class=\"col5\"> 5 </td></tr>\n<tr class=\"row2\"><td class=\"col1\"> 6 </td></tr>\n",
                            '{% tablerow n in numbers cols:5%} {{n}} {% endtablerow %}',

test/liquid/tags/include_tag_test.rb

@@ -39,6 +39,15 @@ class OtherFileSystem
   end
 end
 
+class CountingFileSystem
+  attr_reader :count
+  def read_template_file(template_path, context)
+    @count ||= 0
+    @count += 1
+    'from CountingFileSystem'
+  end
+end
+
 class IncludeTagTest < Test::Unit::TestCase
   include Liquid
 
@@ -136,4 +145,22 @@ class IncludeTagTest < Test::Unit::TestCase
 
     assert_equal "Product: Draft 151cm ", Template.parse("{% include template for product %}").render("template" => 'product', 'product' => { 'title' => 'Draft 151cm'})
   end
-end # IncludeTagTest
+
+  def test_include_tag_caches_second_read_of_same_partial
+    file_system = CountingFileSystem.new
+    assert_equal 'from CountingFileSystemfrom CountingFileSystem',
+      Template.parse("{% include 'pick_a_source' %}{% include 'pick_a_source' %}").render({}, :registers => {:file_system => file_system})
+    assert_equal 1, file_system.count
+  end
+
+  def test_include_tag_doesnt_cache_partials_across_renders
+    file_system = CountingFileSystem.new
+    assert_equal 'from CountingFileSystem',
+      Template.parse("{% include 'pick_a_source' %}").render({}, :registers => {:file_system => file_system})
+    assert_equal 1, file_system.count
+
+    assert_equal 'from CountingFileSystem',
+      Template.parse("{% include 'pick_a_source' %}").render({}, :registers => {:file_system => file_system})
+    assert_equal 2, file_system.count
+  end
+end # IncludeTagTest

test/liquid/variable_test.rb

@@ -11,67 +11,77 @@ class VariableTest < Test::Unit::TestCase
   def test_filters
     var = Variable.new('hello | textileze')
     assert_equal 'hello', var.name
-    assert_equal [[:textileze,[]]], var.filters
+    assert_equal [["textileze",[]]], var.filters
 
     var = Variable.new('hello | textileze | paragraph')
     assert_equal 'hello', var.name
-    assert_equal [[:textileze,[]], [:paragraph,[]]], var.filters
+    assert_equal [["textileze",[]], ["paragraph",[]]], var.filters
 
     var = Variable.new(%! hello | strftime: '%Y'!)
     assert_equal 'hello', var.name
-    assert_equal [[:strftime,["'%Y'"]]], var.filters
+    assert_equal [["strftime",["'%Y'"]]], var.filters
 
     var = Variable.new(%! 'typo' | link_to: 'Typo', true !)
     assert_equal %!'typo'!, var.name
-    assert_equal [[:link_to,["'Typo'", "true"]]], var.filters
+    assert_equal [["link_to",["'Typo'", "true"]]], var.filters
 
     var = Variable.new(%! 'typo' | link_to: 'Typo', false !)
     assert_equal %!'typo'!, var.name
-    assert_equal [[:link_to,["'Typo'", "false"]]], var.filters
+    assert_equal [["link_to",["'Typo'", "false"]]], var.filters
 
     var = Variable.new(%! 'foo' | repeat: 3 !)
     assert_equal %!'foo'!, var.name
-    assert_equal [[:repeat,["3"]]], var.filters
+    assert_equal [["repeat",["3"]]], var.filters
 
     var = Variable.new(%! 'foo' | repeat: 3, 3 !)
     assert_equal %!'foo'!, var.name
-    assert_equal [[:repeat,["3","3"]]], var.filters
+    assert_equal [["repeat",["3","3"]]], var.filters
 
     var = Variable.new(%! 'foo' | repeat: 3, 3, 3 !)
     assert_equal %!'foo'!, var.name
-    assert_equal [[:repeat,["3","3","3"]]], var.filters
+    assert_equal [["repeat",["3","3","3"]]], var.filters
 
     var = Variable.new(%! hello | strftime: '%Y, okay?'!)
     assert_equal 'hello', var.name
-    assert_equal [[:strftime,["'%Y, okay?'"]]], var.filters
+    assert_equal [["strftime",["'%Y, okay?'"]]], var.filters
 
     var = Variable.new(%! hello | things: "%Y, okay?", 'the other one'!)
     assert_equal 'hello', var.name
-    assert_equal [[:things,["\"%Y, okay?\"","'the other one'"]]], var.filters
+    assert_equal [["things",["\"%Y, okay?\"","'the other one'"]]], var.filters
+  end
+
+  def test_utf8_filters
+    var = Variable.new("foo | chinese\u6000filter: value\u6000")
+    assert_equal 'foo', var.name
+    assert_equal [["chinese\u6000filter",["value\u6000"]]], var.filters
   end
 
   def test_filter_with_date_parameter
 
     var = Variable.new(%! '2006-06-06' | date: "%m/%d/%Y"!)
     assert_equal "'2006-06-06'", var.name
-    assert_equal [[:date,["\"%m/%d/%Y\""]]], var.filters
+    assert_equal [["date",["\"%m/%d/%Y\""]]], var.filters
 
   end
 
   def test_filters_without_whitespace
     var = Variable.new('hello | textileze | paragraph')
     assert_equal 'hello', var.name
-    assert_equal [[:textileze,[]], [:paragraph,[]]], var.filters
+    assert_equal [["textileze",[]], ["paragraph",[]]], var.filters
 
     var = Variable.new('hello|textileze|paragraph')
     assert_equal 'hello', var.name
-    assert_equal [[:textileze,[]], [:paragraph,[]]], var.filters
+    assert_equal [["textileze",[]], ["paragraph",[]]], var.filters
+
+    var = Variable.new("hello|replace:'foo','bar'|textileze")
+    assert_equal 'hello', var.name
+    assert_equal [["replace", ["'foo'", "'bar'"]], ["textileze", []]], var.filters
   end
 
   def test_symbol
     var = Variable.new("http://disney.com/logo.gif | image: 'med' ")
     assert_equal 'http://disney.com/logo.gif', var.name
-    assert_equal [[:image,["'med'"]]], var.filters
+    assert_equal [["image",["'med'"]]], var.filters
   end
 
   def test_string_single_quoted
@@ -103,6 +113,12 @@ class VariableTest < Test::Unit::TestCase
     var = Variable.new(%| test.test |)
     assert_equal 'test.test', var.name
   end
+
+  def test_filter_with_keyword_arguments
+    var = Variable.new(%! hello | things: greeting: "world", farewell: 'goodbye'!)
+    assert_equal 'hello', var.name
+    assert_equal [['things',["greeting: \"world\"","farewell: 'goodbye'"]]], var.filters
+  end
 end