Use StringSlice.join on the rendered results.

ext/liquid/block.c

@@ -1,7 +1,7 @@
 #include "liquid_ext.h"
 
 VALUE cLiquidBlock;
-ID intern_assert_missing_delimitation, intern_block_delimiter, intern_is_blank, intern_new,
+ID intern_assert_missing_delimitation, intern_block_delimiter, intern_is_blank,
    intern_new_with_options, intern_tags, intern_unknown_tag, intern_unterminated_tag,
    intern_unterminated_variable;
 
@@ -133,7 +133,7 @@ static VALUE rb_parse_body(VALUE self, VALUE tokenizerObj)
             break;
         }
         case TOKEN_STRING:
-            rb_ary_push(nodelist, rb_str_new(token.str, token.length));
+            rb_ary_push(nodelist, liquid_string_slice_new(token.str, token.length));
             if (blank) {
                 int i;
                 for (i = 0; i < token.length; i++) {
@@ -156,7 +156,6 @@ void init_liquid_block()
     intern_assert_missing_delimitation = rb_intern("assert_missing_delimitation!");
     intern_block_delimiter = rb_intern("block_delimiter");
     intern_is_blank = rb_intern("blank?");
-    intern_new = rb_intern("new");
     intern_new_with_options = rb_intern("new_with_options");
     intern_tags = rb_intern("tags");
     intern_unknown_tag = rb_intern("unknown_tag");

ext/liquid/liquid_ext.c

@@ -2,9 +2,11 @@
 
 VALUE mLiquid;
 VALUE cLiquidTemplate, cLiquidTag, cLiquidVariable;
+ID intern_new;
 
 void Init_liquid(void)
 {
+    intern_new = rb_intern("new");
     mLiquid = rb_define_module("Liquid");
     cLiquidTemplate = rb_define_class_under(mLiquid, "Template", rb_cObject);
     cLiquidTag = rb_define_class_under(mLiquid, "Tag", rb_cObject);
@@ -12,5 +14,5 @@ void Init_liquid(void)
 
     init_liquid_tokenizer();
     init_liquid_block();
-    init_liquid_variable();
+    init_liquid_string_slice();
 }

ext/liquid/liquid_ext.h

@@ -7,9 +7,10 @@
 
 #include "tokenizer.h"
 #include "block.h"
+#include "slice.h"
 #include "utils.h"
-#include "variable.h"
 
+extern ID intern_new;
 extern VALUE mLiquid;
 extern VALUE cLiquidTemplate, cLiquidTag, cLiquidVariable;
 

ext/liquid/slice.c

@@ -0,0 +1,167 @@
+#include "liquid_ext.h"
+
+VALUE cLiquidStringSlice;
+
+static void mark_slice(void *ptr)
+{
+    if (!ptr)
+        return;
+    struct string_slice *slice = ptr;
+    rb_gc_mark(slice->source);
+}
+
+static void free_slice(void *ptr)
+{
+    struct string_slice *slice = ptr;
+    xfree(slice);
+}
+
+VALUE liquid_string_slice_new(const char *str, long length)
+{
+    return rb_funcall(cLiquidStringSlice, intern_new, 3, rb_str_new(str, length), INT2FIX(0), INT2FIX(length));
+}
+
+static VALUE rb_allocate(VALUE klass)
+{
+    struct string_slice *slice;
+    VALUE obj = Data_Make_Struct(klass, struct string_slice, mark_slice, free_slice, slice);
+    return obj;
+}
+
+static VALUE rb_initialize(VALUE self, VALUE source, VALUE offset_value, VALUE length_value)
+{
+    long offset = rb_fix2int(offset_value);
+    long length = rb_fix2int(length_value);
+    if (length < 0)
+        rb_raise(rb_eArgError, "negative string length");
+    if (offset < 0)
+        rb_raise(rb_eArgError, "negative string offset");
+
+    if (TYPE(source) == T_DATA && RBASIC_CLASS(source) == cLiquidStringSlice) {
+        struct string_slice *source_slice = DATA_PTR(source);
+        source = source_slice->source;
+        offset += source_slice->str - RSTRING_PTR(source);
+    } else {
+        source = rb_string_value(&source);
+        source = rb_str_dup_frozen(source);
+    }
+
+    struct string_slice *slice;
+    Data_Get_Struct(self, struct string_slice, slice);
+    slice->source = source;
+    slice->str = RSTRING_PTR(source) + offset;
+    slice->length = length;
+    if (length > RSTRING_LEN(source) - offset)
+        rb_raise(rb_eArgError, "slice bounds outside source string bounds");
+
+    return Qnil;
+}
+
+static VALUE rb_slice_to_str(VALUE self)
+{
+    struct string_slice *slice;
+    Data_Get_Struct(self, struct string_slice, slice);
+
+    VALUE source = slice->source;
+    if (slice->str == RSTRING_PTR(source) && slice->length == RSTRING_LEN(source))
+        return source;
+
+    source = rb_str_new(slice->str, slice->length);
+    slice->source = source;
+    slice->str = RSTRING_PTR(source);
+    return source;
+}
+
+static VALUE rb_slice_slice(VALUE self, VALUE offset, VALUE length)
+{
+    return rb_funcall(cLiquidStringSlice, intern_new, 3, self, offset, length);
+}
+
+static VALUE rb_slice_length(VALUE self)
+{
+    struct string_slice *slice;
+    Data_Get_Struct(self, struct string_slice, slice);
+    return INT2FIX(slice->length);
+}
+
+static VALUE rb_slice_equal(VALUE self, VALUE other)
+{
+    struct string_slice *this_slice;
+    Data_Get_Struct(self, struct string_slice, this_slice);
+
+    const char *other_str;
+    long other_length;
+    if (TYPE(other) == T_DATA && RBASIC_CLASS(other) == cLiquidStringSlice) {
+        struct string_slice *other_slice = DATA_PTR(other);
+        other_str = other_slice->str;
+        other_length = other_slice->length;
+    } else {
+        other = rb_string_value(&other);
+        other_length = RSTRING_LEN(other);
+        other_str = RSTRING_PTR(other);
+    }
+    bool equal = this_slice->length == other_length && !memcmp(this_slice->str, other_str, other_length);
+    return equal ? Qtrue : Qfalse;
+}
+
+static VALUE rb_slice_inspect(VALUE self)
+{
+    VALUE quoted = rb_str_inspect(rb_slice_to_str(self));
+    return rb_sprintf("#<Liquid::StringSlice: %.*s>", (int)RSTRING_LEN(quoted), RSTRING_PTR(quoted));
+}
+
+static VALUE rb_slice_join(VALUE klass, VALUE ary)
+{
+    ary = rb_ary_to_ary(ary);
+
+    long i;
+    long result_length = 0;
+    for (i = 0; i < RARRAY_LEN(ary); i++) {
+        VALUE element = RARRAY_AREF(ary, i);
+
+        if (TYPE(element) == T_DATA && RBASIC_CLASS(element) == cLiquidStringSlice) {
+            struct string_slice *slice = DATA_PTR(element);
+            result_length += slice->length;
+        } else if (TYPE(element) == T_STRING) {
+            result_length += RSTRING_LEN(element);
+        }
+    }
+
+    VALUE result = rb_str_buf_new(result_length);
+    for (i = 0; i < RARRAY_LEN(ary); i++) {
+        VALUE element = RARRAY_AREF(ary, i);
+
+        const char *element_string;
+        long element_length;
+        if (TYPE(element) == T_DATA && RBASIC_CLASS(element) == cLiquidStringSlice) {
+            struct string_slice *slice = DATA_PTR(element);
+            element_string = slice->str;
+            element_length = slice->length;
+        } else if (NIL_P(element)) {
+            continue;
+        } else {
+            element = rb_check_string_type(element);
+            if (NIL_P(element))
+                continue;
+            element_string = RSTRING_PTR(element);
+            element_length = RSTRING_LEN(element);
+        }
+        rb_str_buf_cat(result, element_string, element_length);
+    }
+    return result;
+}
+
+void init_liquid_string_slice()
+{
+    cLiquidStringSlice = rb_define_class_under(mLiquid, "StringSlice", rb_cObject);
+    rb_define_singleton_method(cLiquidStringSlice, "join", rb_slice_join, 1);
+    rb_define_alloc_func(cLiquidStringSlice, rb_allocate);
+    rb_define_method(cLiquidStringSlice, "initialize", rb_initialize, 3);
+    rb_define_method(cLiquidStringSlice, "==", rb_slice_equal, 1);
+    rb_define_method(cLiquidStringSlice, "length", rb_slice_length, 0);
+    rb_define_alias(cLiquidStringSlice,  "size", "length");
+    rb_define_method(cLiquidStringSlice, "slice", rb_slice_slice, 2);
+    rb_define_method(cLiquidStringSlice, "to_str", rb_slice_to_str, 0);
+    rb_define_alias(cLiquidStringSlice,  "to_s", "to_str");
+    rb_define_method(cLiquidStringSlice, "inspect", rb_slice_inspect, 0);
+}

ext/liquid/slice.h

@@ -0,0 +1,18 @@
+#ifndef LIQUID_SLICE_H
+#define LIQUID_SLICE_H
+
+extern VALUE cLiquidStringSlice;
+
+struct string_slice {
+    VALUE source;
+    const char *str;
+    long length;
+};
+
+VALUE liquid_string_slice_new(const char *str, long length);
+
+void init_liquid_string_slice();
+
+#define STRING_SLICE_GET_STRUCT(obj) ((struct string_slice *)obj_get_data_ptr(obj, cLiquidStringSlice))
+
+#endif

ext/liquid/tokenizer.c

@@ -21,7 +21,7 @@ static VALUE rb_initialize(VALUE self, VALUE source)
 {
     struct liquid_tokenizer *tokenizer;
 
-    Check_Type(source, T_STRING);
+    source = rb_string_value(&source);
     Data_Get_Struct(self, struct liquid_tokenizer, tokenizer);
     tokenizer->cursor = RSTRING_PTR(source);
     tokenizer->length = RSTRING_LEN(source);

ext/liquid/variable.c

@@ -1,179 +0,0 @@
-#include "liquid_ext.h"
-
-VALUE cLiquidVariable;
-extern VALUE mLiquid;
-
-static void free_variable(void *ptr)
-{
-    struct liquid_variable *variable = ptr;
-    xfree(variable);
-}
-
-static VALUE rb_variable_allocate(VALUE klass)
-{
-    VALUE obj;
-    struct liquid_variable *variable;
-
-    obj = Data_Make_Struct(klass, struct liquid_variable, NULL, free_variable, variable);
-    return obj;
-}
-
-static inline int skip_whitespace(char * str, int len) 
-{
-  int skipped = 0; char * ptr = str;
-  while (skipped < len && isspace(*ptr))
-    {skipped++; ptr++;}
-  return skipped;
-}
-
-static char * get_quoted_fragment(char * str, int len, int * ret_size, int * end_offset, bool colon) 
-{
-  int p = 0; /* Current position in string */
-  int start = -1, end = -1; /* Start and end indices for the found string */
-  char quoted_by = -1; /* Is the current part of string quoted by a single or double quote? If so 
-                          ignore any special chars */
-
-  while (p < len) {
-    
-    switch (str[p]) {
-      case '"': 
-        if (start == -1) {start = p; quoted_by = '"';}
-        else if (str[start] == '"') {end = p; goto quoted_fragment_found;}
-        else if (quoted_by == -1) quoted_by = '"';
-        else if (quoted_by == '"') quoted_by = -1;
-        break;
-      case '\'':
-        if (start == -1) {start = p; quoted_by = '\'';}
-        else if (str[start] == '\'') {end = p; goto quoted_fragment_found;}
-        else if (quoted_by == -1) quoted_by = '\'';
-        else if (quoted_by == '\'') quoted_by = -1;
-        break;
-      case ':': 
-        if (colon)
-          if (start != -1 && quoted_by == -1) {end = p-1; goto quoted_fragment_found;} 
-        else
-          if (start == -1) start = p;
-        break;
-      case '|':
-      case ',':
-      case '\n':
-      case '\r':
-      case '\f':
-      case '\t':
-      case '\v':
-      case ' ': 
-        if (start != -1 && quoted_by == -1) {end = p-1; goto quoted_fragment_found;} 
-        break;
-      default: 
-        if (start == -1) start = p; 
-        break;
-    }
-    p++;
-  }
-  if (p == len && start != -1 && end == -1) end = len-1;
-
-quoted_fragment_found:
-  if (end >= start) {
-    *ret_size = end-start+1;
-    *end_offset = end+1;
-    return &str[start];
-  } else {
-    *ret_size = 0;
-    return NULL;
-  }
-}
-
-static VALUE get_filters(char * str, int len, VALUE self) {
-  VALUE filters_arr = rb_ary_new(); 
-
-  int p = 0; 
-  int ret_size, end_offset; 
-  char * f;
-
-  while(p<len) {
-    if (str[p] == '|') {
-      VALUE filter = rb_ary_new();
-      VALUE f_args = rb_ary_new();
-
-      p += skip_whitespace(&str[p+1], len-p-1);
-      f = get_quoted_fragment(&str[p], len-p, &ret_size, &end_offset, true);
-      p += end_offset;
-
-      if (f) {
-        if (f[ret_size-1] == ':') ret_size--;
-        rb_ary_push(filter, rb_str_new(f, ret_size));
-      }
-
-      /* Check for filter arguments */
-      do {
-        if (p<len) {
-          p += skip_whitespace(&str[p], len-p);
-
-          // printf("\n1. %.*s\n", len-p, &str[p]);
-
-          if (str[p] != '|') {
-            f = get_quoted_fragment(&str[p], len-p, &ret_size, &end_offset, false);
-
-            // printf("\n2. %.*s\n", ret_size, f);
-            
-            p += end_offset;
-            p += skip_whitespace(&str[p], len-p);
-
-            if (str[p] == '|') p--;
-
-            if (f) rb_ary_push(f_args, rb_str_new(f, ret_size));
-          
-          } else p--;
-        }
-
-      } while (str[p] == ',' || str[p] == ':');
-
-      rb_ary_push(filter, f_args);
-
-      /* Add to filters_arr array */
-      rb_ary_push(filters_arr, filter);
-    }
-    p++;
-  }
-  return filters_arr;
-}
-
-static VALUE rb_variable_lax_parse(VALUE self, VALUE m) 
-{
-  char * markup = RSTRING_PTR(m);
-  int markup_len = RSTRING_LEN(m);
-
-  char * cursor = markup; int cursor_pos = 0; 
-  VALUE filters_arr; 
-  int size, end_offset;
-
-  /* Extract name */
-  cursor_pos += skip_whitespace(markup, markup_len); 
-  cursor = markup + cursor_pos;
-  cursor = get_quoted_fragment(cursor, markup_len - cursor_pos, &size, &end_offset, false);
-
-  if (cursor == NULL) {
-    rb_iv_set(self, "@name", Qnil);
-    filters_arr = rb_ary_new(); 
-    rb_iv_set(self, "@filters", filters_arr);
-  }
-  else 
-  {
-    rb_iv_set(self, "@name", rb_str_new(cursor, size));
-
-    /* Extract filters */
-    if (end_offset < markup_len) {
-      cursor = &markup[end_offset];
-      filters_arr = get_filters(cursor, markup_len - end_offset, self);
-      rb_iv_set(self, "@filters", filters_arr);
-    }
-  }
-  return filters_arr;
-}
-
-void init_liquid_variable()
-{
-    cLiquidVariable = rb_define_class_under(mLiquid, "Variable", rb_cObject);
-    rb_define_alloc_func(cLiquidVariable, rb_variable_allocate);
-    rb_define_method(cLiquidVariable, "lax_parse", rb_variable_lax_parse, 1);
-}

ext/liquid/variable.h

@@ -1,13 +0,0 @@
-#ifndef LIQUID_VARIABLE_H
-#define LIQUID_VARIABLE_H
-
-#include <regex.h>
-
-struct liquid_variable {
-    char *markup; long markup_len;
-    char *name; long name_len;
-};
-
-void init_liquid_variable();
-
-#endif

lib/liquid/block.rb

@@ -95,7 +95,7 @@ module Liquid
         end
       end
 
-      output.join
+      StringSlice.join(output)
     end
   end
 end

lib/liquid/context.rb

@@ -29,7 +29,7 @@ module Liquid
     end
 
     def increment_used_resources(key, obj)
-      @resource_limits[key] += if obj.kind_of?(String) || obj.kind_of?(Array) || obj.kind_of?(Hash)
+      @resource_limits[key] += if obj.kind_of?(StringSlice) || obj.kind_of?(String) || obj.kind_of?(Array) || obj.kind_of?(Hash)
         obj.length
       else
         1

lib/liquid/variable.rb

@@ -11,6 +11,7 @@ module Liquid
   #   {{ user | link }}
   #
   class Variable
+    FilterParser = /(?:#{FilterSeparator}|(?:\s*(?:#{QuotedFragment}|#{ArgumentSeparator})\s*)+)/o
     EasyParse = /\A *(\w+(?:\.\w+)*) *\z/
     attr_accessor :filters, :name, :warnings
 
@@ -34,22 +35,22 @@ module Liquid
       end
     end
 
-#     def lax_parse(markup)
-#       @filters = []
-#       if match = markup.match(/\s*(#{QuotedFragment})(.*)/o)
-#         @name = match[1]
-#         if match[2].match(/#{FilterSeparator}\s*(.*)/o)
-#           filters = Regexp.last_match(1).scan(FilterParser)
-#           filters.each do |f|
-#             if matches = f.match(/\s*(\w+)/)
-#               filtername = matches[1]
-#               filterargs = f.scan(/(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o).flatten
-#               @filters << [filtername, filterargs]
-#             end
-#           end
-#         end
-#       end
-#     end
+    def lax_parse(markup)
+      @filters = []
+      if match = markup.match(/\s*(#{QuotedFragment})(.*)/o)
+        @name = match[1]
+        if match[2].match(/#{FilterSeparator}\s*(.*)/o)
+          filters = Regexp.last_match(1).scan(FilterParser)
+          filters.each do |f|
+            if matches = f.match(/\s*(\w+)/)
+              filtername = matches[1]
+              filterargs = f.scan(/(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o).flatten
+              @filters << [filtername, filterargs]
+            end
+          end
+        end
+      end
+    end
 
     def strict_parse(markup)
       # Very simple valid cases

test/liquid/block_test.rb

@@ -12,34 +12,34 @@ class BlockTest < Test::Unit::TestCase
     template = Liquid::Template.parse("{{funk}}  ")
     assert_equal 2, template.root.nodelist.size
     assert_equal Variable, template.root.nodelist[0].class
-    assert_equal String, template.root.nodelist[1].class
+    assert_equal StringSlice, template.root.nodelist[1].class
   end
 
   def test_variable_end
     template = Liquid::Template.parse("  {{funk}}")
     assert_equal 2, template.root.nodelist.size
-    assert_equal String, template.root.nodelist[0].class
+    assert_equal StringSlice, template.root.nodelist[0].class
     assert_equal Variable, template.root.nodelist[1].class
   end
 
   def test_variable_middle
     template = Liquid::Template.parse("  {{funk}}  ")
     assert_equal 3, template.root.nodelist.size
-    assert_equal String, template.root.nodelist[0].class
+    assert_equal StringSlice, template.root.nodelist[0].class
     assert_equal Variable, template.root.nodelist[1].class
-    assert_equal String, template.root.nodelist[2].class
+    assert_equal StringSlice, template.root.nodelist[2].class
   end
 
   def test_variable_many_embedded_fragments
     template = Liquid::Template.parse("  {{funk}} {{so}} {{brother}} ")
     assert_equal 7, template.root.nodelist.size
-    assert_equal [String, Variable, String, Variable, String, Variable, String],
+    assert_equal [StringSlice, Variable, StringSlice, Variable, StringSlice, Variable, StringSlice],
                  block_types(template.root.nodelist)
   end
 
   def test_with_block
     template = Liquid::Template.parse("  {% comment %} {% endcomment %} ")
-    assert_equal [String, Comment, String], block_types(template.root.nodelist)
+    assert_equal [StringSlice, Comment, StringSlice], block_types(template.root.nodelist)
     assert_equal 3, template.root.nodelist.size
   end
 

test/liquid/parsing_quirks_test.rb

@@ -8,7 +8,7 @@ class ParsingQuirksTest < Test::Unit::TestCase
     template = Template.parse(text)
 
     assert_equal text, template.render
-    assert_equal [String], template.root.nodelist.collect {|i| i.class}
+    assert_equal [StringSlice], template.root.nodelist.collect {|i| i.class}
   end
 
   def test_raise_on_single_close_bracet

test/liquid/string_slice_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+class StringSliceTest < Test::Unit::TestCase
+  def test_new_from_string
+    assert_equal 'slice', Liquid::StringSlice.new("slice and dice", 0, 5).to_str
+    assert_equal 'and', Liquid::StringSlice.new("slice and dice", 6, 3).to_str
+    assert_equal 'dice', Liquid::StringSlice.new("slice and dice", 10, 4).to_str
+    assert_equal 'slice and dice', Liquid::StringSlice.new("slice and dice", 0, 14).to_str
+  end
+
+  def test_new_from_slice
+    slice1 = Liquid::StringSlice.new("slice and dice", 0, 14)
+    slice2 = Liquid::StringSlice.new(slice1, 6, 8)
+    slice3 = Liquid::StringSlice.new(slice2, 0, 3)
+    assert_equal "slice and dice", slice1.to_str
+    assert_equal "and dice", slice2.to_str
+    assert_equal "and", slice3.to_str
+  end
+
+  def test_slice
+    slice = Liquid::StringSlice.new("slice and dice", 2, 10)
+    assert_equal "and", slice.slice(4, 3).to_str
+  end
+
+  def test_length
+    slice = Liquid::StringSlice.new("slice and dice", 6, 3)
+    assert_equal 3, slice.length
+    assert_equal 3, slice.size
+  end
+
+  def test_equal
+    assert_equal 'and', Liquid::StringSlice.new("slice and dice", 6, 3)
+  end
+end

test/liquid/variable_test.rb

@@ -51,9 +51,11 @@ class VariableTest < Test::Unit::TestCase
   end
 
   def test_filter_with_date_parameter
+
     var = Variable.new(%! '2006-06-06' | date: "%m/%d/%Y"!)
     assert_equal "'2006-06-06'", var.name
     assert_equal [["date",["\"%m/%d/%Y\""]]], var.filters
+
   end
 
   def test_filters_without_whitespace
@@ -71,7 +73,7 @@ class VariableTest < Test::Unit::TestCase
   end
 
   def test_symbol
-    var = Variable.new("http://disney.com/logo.gif | image: 'med'", :error_mode => :lax)
+    var = Variable.new("http://disney.com/logo.gif | image: 'med' ", :error_mode => :lax)
     assert_equal "http://disney.com/logo.gif", var.name
     assert_equal [["image",["'med'"]]], var.filters
   end