Compare commits

..

1 Commits

Author SHA1 Message Date
Mike Angell
cf5a42245f Remove jruby and truffleruby testing 2019-09-20 02:01:58 +10:00
31 changed files with 94 additions and 294 deletions

View File

@@ -8,7 +8,6 @@ Performance:
Enabled: true
AllCops:
TargetRubyVersion: 2.4
Exclude:
- 'vendor/bundle/**/*'

View File

@@ -6,6 +6,26 @@
# Note that changes in the inspected code, or installation of new
# versions of RuboCop, may require this file to be generated again.
# Offense count: 2
Lint/AmbiguousOperator:
Exclude:
- 'test/unit/condition_unit_test.rb'
# Offense count: 21
# Configuration parameters: AllowSafeAssignment.
Lint/AssignmentInCondition:
Exclude:
- 'lib/liquid/block_body.rb'
- 'lib/liquid/lexer.rb'
- 'lib/liquid/standardfilters.rb'
- 'lib/liquid/tags/for.rb'
- 'lib/liquid/tags/if.rb'
- 'lib/liquid/tags/raw.rb'
- 'lib/liquid/variable.rb'
- 'performance/profile.rb'
- 'test/test_helper.rb'
- 'test/unit/tokenizer_unit_test.rb'
# Offense count: 2
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
@@ -14,6 +34,17 @@ Lint/InheritException:
Exclude:
- 'lib/liquid/interrupts.rb'
# Offense count: 2
Lint/UselessAssignment:
Exclude:
- 'performance/shopify/database.rb'
# Offense count: 1
# Configuration parameters: CheckForMethodsWithNoSideEffects.
Lint/Void:
Exclude:
- 'lib/liquid/parse_context.rb'
# Offense count: 98
# Cop supports --auto-correct.
# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
@@ -45,4 +76,19 @@ Style/ClassVars:
Exclude:
- 'lib/liquid/condition.rb'
- 'lib/liquid/strainer.rb'
- 'lib/liquid/template.rb'
- 'lib/liquid/template.rb'
# Offense count: 1
# Configuration parameters: AllowCoercion.
Style/DateTime:
Exclude:
- 'test/unit/context_unit_test.rb'
# Offense count: 9
# Cop supports --auto-correct.
# Configuration parameters: AllowAsExpressionSeparator.
Style/Semicolon:
Exclude:
- 'test/integration/error_handling_test.rb'
- 'test/integration/template_test.rb'
- 'test/unit/context_unit_test.rb'

View File

@@ -78,10 +78,8 @@ require 'liquid/tokenizer'
require 'liquid/parse_context'
require 'liquid/partial_cache'
require 'liquid/usage'
require 'liquid/register'
require 'liquid/static_registers'
# Load all the tags of the standard library
#
Dir["#{__dir__}/liquid/tags/*.rb"].each { |f| require f }
Dir["#{__dir__}/liquid/registers/*.rb"].each { |f| require f }

View File

@@ -27,7 +27,7 @@ module Liquid
end
private def parse_for_liquid_tag(tokenizer, parse_context)
while (token = tokenizer.shift)
while token = tokenizer.shift
unless token.empty? || token =~ WhitespaceOrNothing
unless token =~ LiquidTagToken
# line isn't empty but didn't match tag syntax, yield and let the
@@ -36,7 +36,7 @@ module Liquid
end
tag_name = Regexp.last_match(1)
markup = Regexp.last_match(2)
unless (tag = registered_tags[tag_name])
unless tag = registered_tags[tag_name]
# end parsing if we reach an unknown tag and let the caller decide
# determine how to proceed
return yield tag_name, markup
@@ -52,7 +52,7 @@ module Liquid
end
private def parse_for_document(tokenizer, parse_context, &block)
while (token = tokenizer.shift)
while token = tokenizer.shift
next if token.empty?
case
when token.start_with?(TAGSTART)
@@ -74,7 +74,7 @@ module Liquid
next parse_for_liquid_tag(liquid_tag_tokenizer, parse_context, &block)
end
unless (tag = registered_tags[tag_name])
unless tag = registered_tags[tag_name]
# end parsing if we reach an unknown tag and let the caller decide
# determine how to proceed
return yield tag_name, markup
@@ -122,7 +122,7 @@ module Liquid
context.resource_limits.render_score += @nodelist.length
idx = 0
while (node = @nodelist[idx])
while node = @nodelist[idx]
previous_output_size = output.bytesize
case node
@@ -154,13 +154,7 @@ module Liquid
private
def render_node(context, output, node)
if node.disabled?(context)
output << node.disabled_error_message
return
end
disable_tags(context, node.disabled_tags) do
node.render_to_output_buffer(context, output)
end
node.render_to_output_buffer(context, output)
rescue UndefinedVariable, UndefinedDropMethod, UndefinedFilter => e
context.handle_error(e, node.line_number)
rescue ::StandardError => e
@@ -168,11 +162,6 @@ module Liquid
output << context.handle_error(e, line_number)
end
def disable_tags(context, tags, &block)
return yield if tags.empty?
context.registers[:disabled_tags].disable(tags, &block)
end
def raise_if_resource_limits_reached(context, length)
context.resource_limits.render_length += length
return unless context.resource_limits.reached?

View File

@@ -59,7 +59,7 @@ module Liquid
end
def full_path(template_path)
raise FileSystemError, "Illegal template name '#{template_path}'" unless %r{\A[^./][a-zA-Z0-9_/]+\z}.match?(template_path)
raise FileSystemError, "Illegal template name '#{template_path}'" unless template_path =~ %r{\A[^./][a-zA-Z0-9_/]+\z}
full_path = if template_path.include?('/')
File.join(root, File.dirname(template_path), @pattern % File.basename(template_path))

View File

@@ -33,21 +33,15 @@ module Liquid
until @ss.eos?
@ss.skip(WHITESPACE_OR_NOTHING)
break if @ss.eos?
tok = if (t = @ss.scan(COMPARISON_OPERATOR))
[:comparison, t]
elsif (t = @ss.scan(SINGLE_STRING_LITERAL))
[:string, t]
elsif (t = @ss.scan(DOUBLE_STRING_LITERAL))
[:string, t]
elsif (t = @ss.scan(NUMBER_LITERAL))
[:number, t]
elsif (t = @ss.scan(IDENTIFIER))
[:id, t]
elsif (t = @ss.scan(DOTDOT))
[:dotdot, t]
tok = if t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t]
elsif t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t]
elsif t = @ss.scan(DOUBLE_STRING_LITERAL) then [:string, t]
elsif t = @ss.scan(NUMBER_LITERAL) then [:number, t]
elsif t = @ss.scan(IDENTIFIER) then [:id, t]
elsif t = @ss.scan(DOTDOT) then [:dotdot, t]
else
c = @ss.getch
if (s = SPECIALS[c])
if s = SPECIALS[c]
[s, c]
else
raise SyntaxError, "Unexpected character #{c}"

View File

@@ -25,5 +25,3 @@
render: "Syntax error in tag 'render' - Template name must be a quoted string"
argument:
include: "Argument error in tag 'include' - Illegal template name"
disabled:
tag: "usage is not allowed in this context"

View File

@@ -21,6 +21,7 @@ module Liquid
@partial = value
@options = value ? partial_options : @template_options
@error_mode = @options[:error_mode] || Template.error_mode
value
end
def partial_options

View File

@@ -1,6 +0,0 @@
# frozen_string_literal: true
module Liquid
class Register
end
end

View File

@@ -1,32 +0,0 @@
# frozen_string_literal: true
module Liquid
class DisabledTags < Register
def initialize
@disabled_tags = {}
end
def disabled?(tag)
@disabled_tags.key?(tag) && @disabled_tags[tag] > 0
end
def disable(tags)
tags.each(&method(:increment))
yield
ensure
tags.each(&method(:decrement))
end
private
def increment(tag)
@disabled_tags[tag] ||= 0
@disabled_tags[tag] += 1
end
def decrement(tag)
@disabled_tags[tag] -= 1
end
end
Template.add_register(:disabled_tags, DisabledTags.new)
end

View File

@@ -193,23 +193,6 @@ module Liquid
end
end
# Sort elements of an array in numeric order
# provide optional property with which to sort an array of hashes or drops
def sort_numeric(input, property = nil)
ary = InputIterator.new(input)
if property.nil?
ary.sort do |a, b|
Utils.to_number(a) <=> Utils.to_number(b)
end
elsif ary.empty? # The next two cases assume a non-empty array.
[]
elsif ary.first.respond_to?(:[]) && !ary.first[property].nil?
ary.sort do |a, b|
Utils.to_number(a[property]) <=> Utils.to_number(b[property])
end
end
end
# Remove duplicate elements from an array
# provide optional property with which to determine uniqueness
def uniq(input, property = nil)
@@ -344,7 +327,7 @@ module Liquid
def date(input, format)
return input if format.to_s.empty?
return input unless (date = Utils.to_date(input))
return input unless date = Utils.to_date(input)
date.strftime(format.to_s)
end

View File

@@ -13,15 +13,7 @@ module Liquid
tag
end
def disable_tags(*tags)
disabled_tags.push(*tags)
end
private :new
def disabled_tags
@disabled_tags ||= []
end
end
def initialize(tag_name, markup, parse_context)
@@ -46,14 +38,6 @@ module Liquid
''
end
def disabled?(context)
context.registers[:disabled_tags].disabled?(tag_name)
end
def disabled_error_message
"#{tag_name} #{options[:locale].t('errors.disabled.tag')}"
end
# For backwards compatibility with custom tags. In a future release, the semantics
# of the `render_to_output_buffer` method will become the default and the `render`
# method will be removed.
@@ -65,9 +49,5 @@ module Liquid
def blank?
false
end
def disabled_tags
self.class.disabled_tags
end
end
end

View File

@@ -111,7 +111,7 @@ module Liquid
@reversed = p.id?('reversed')
while p.look(:id) && p.look(:colon, 1)
unless (attribute = p.id?('limit') || p.id?('offset'))
unless attribute = p.id?('limit') || p.id?('offset')
raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_attribute")
end
p.consume

View File

@@ -94,7 +94,7 @@ module Liquid
def parse_binary_comparisons(p)
condition = parse_comparison(p)
first_condition = condition
while (op = (p.id?('and') || p.id?('or')))
while op = (p.id?('and') || p.id?('or'))
child_condition = parse_comparison(p)
condition.send(op, child_condition)
condition = child_condition
@@ -104,7 +104,7 @@ module Liquid
def parse_comparison(p)
a = Expression.parse(p.expression)
if (op = p.consume?(:comparison))
if op = p.consume?(:comparison)
b = Expression.parse(p.expression)
Condition.new(a, op, b)
else

View File

@@ -13,7 +13,7 @@ module Liquid
def parse(tokens)
@body = +''
while (token = tokens.shift)
while token = tokens.shift
if token =~ FullTokenPossiblyInvalid
@body << Regexp.last_match(1) if Regexp.last_match(1) != ""
return if block_delimiter == Regexp.last_match(2)
@@ -40,7 +40,7 @@ module Liquid
protected
def ensure_valid_markup(tag_name, markup, parse_context)
unless Syntax.match?(markup)
unless markup =~ Syntax
raise SyntaxError, parse_context.locale.t("errors.syntax.tag_unexpected_args", tag: tag_name)
end
end

View File

@@ -4,8 +4,6 @@ module Liquid
class Render < Tag
SYNTAX = /(#{QuotedString})#{QuotedFragment}*/o
disable_tags "include"
attr_reader :template_name_expr, :attributes
def initialize(tag_name, markup, options)
@@ -24,10 +22,6 @@ module Liquid
end
def render_to_output_buffer(context, output)
render_tag(context, output)
end
def render_tag(context, output)
# Though we evaluate this here we will only ever parse it as a string literal.
template_name = context.evaluate(@template_name_expr)
raise ArgumentError, options[:locale].t("errors.argument.include") unless template_name

View File

@@ -92,14 +92,6 @@ module Liquid
@tags ||= TagRegistry.new
end
def add_register(name, klass)
registers[name.to_sym] = klass
end
def registers
@registers ||= {}
end
def error_mode
@error_mode ||= :lax
end
@@ -199,26 +191,18 @@ module Liquid
output = nil
context_register = context.registers.is_a?(StaticRegisters) ? context.registers.static : context.registers
case args.last
when Hash
options = args.pop
output = options[:output] if options[:output]
options[:registers]&.each do |key, register|
context_register[key] = register
end
registers.merge!(options[:registers]) if options[:registers].is_a?(Hash)
apply_options_to_context(context, options)
when Module, Array
context.add_filters(args.pop)
end
Template.registers.each do |key, register|
context_register[key] = register
end
# Retrying a render resets resource usage
context.resource_limits.reset

View File

@@ -52,7 +52,7 @@ module Liquid
when Numeric
obj
when String
/\A-?\d+\.\d+\z/.match?(obj.strip) ? BigDecimal(obj) : obj.to_i
obj.strip =~ /\A-?\d+\.\d+\z/ ? BigDecimal(obj) : obj.to_i
else
if obj.respond_to?(:to_number)
obj.to_number

View File

@@ -104,21 +104,13 @@ module Liquid
output
end
def disabled?(_context)
false
end
def disabled_tags
[]
end
private
def parse_filter_expressions(filter_name, unparsed_args)
filter_args = []
keyword_args = nil
unparsed_args.each do |a|
if (matches = a.match(JustTagAttributes))
if matches = a.match(JustTagAttributes)
keyword_args ||= {}
keyword_args[matches[1]] = Expression.parse(matches[2])
else

View File

@@ -15,7 +15,7 @@ profiler.run
end
end
if profile_type == :cpu && (graph_filename = ENV['GRAPH_FILENAME'])
if profile_type == :cpu && graph_filename = ENV['GRAPH_FILENAME']
File.open(graph_filename, 'w') do |f|
StackProf::Report.new(results).print_graphviz(nil, f)
end

View File

@@ -32,8 +32,8 @@ module Database
db['article'] = db['blog']['articles'].first
db['cart'] = {
'total_price' => db['line_items'].values.inject(0) { |sum, item| sum + item['line_price'] * item['quantity'] },
'item_count' => db['line_items'].values.inject(0) { |sum, item| sum + item['quantity'] },
'total_price' => db['line_items'].values.inject(0) { |sum, item| sum += item['line_price'] * item['quantity'] },
'item_count' => db['line_items'].values.inject(0) { |sum, item| sum += item['quantity'] },
'items' => db['line_items'].values,
}

View File

@@ -211,10 +211,7 @@ class ErrorHandlingTest < Minitest::Test
def test_setting_default_exception_renderer
old_exception_renderer = Liquid::Template.default_exception_renderer
exceptions = []
Liquid::Template.default_exception_renderer = ->(e) {
exceptions << e
''
}
Liquid::Template.default_exception_renderer = ->(e) { exceptions << e; '' }
template = Liquid::Template.parse('This is a runtime error: {{ errors.argument_error }}')
output = template.render('errors' => ErrorDrop.new)
@@ -228,10 +225,7 @@ class ErrorHandlingTest < Minitest::Test
def test_exception_renderer_exposing_non_liquid_error
template = Liquid::Template.parse('This is a runtime error: {{ errors.runtime_error }}', line_numbers: true)
exceptions = []
handler = ->(e) {
exceptions << e
e.cause
}
handler = ->(e) { exceptions << e; e.cause }
output = template.render({ 'errors' => ErrorDrop.new }, exception_renderer: handler)

View File

@@ -1,27 +0,0 @@
# frozen_string_literal: true
require 'test_helper'
class DisabledTagsTest < Minitest::Test
include Liquid
class DisableRaw < Block
disable_tags "raw"
end
class DisableRawEcho < Block
disable_tags "raw", "echo"
end
def test_disables_raw
with_custom_tag('disable', DisableRaw) do
assert_template_result 'raw usage is not allowed in this contextfoo', '{% disable %}{% raw %}Foobar{% endraw %}{% echo "foo" %}{% enddisable %}'
end
end
def test_disables_echo_and_raw
with_custom_tag('disable', DisableRawEcho) do
assert_template_result 'raw usage is not allowed in this contextecho usage is not allowed in this context', '{% disable %}{% raw %}Foobar{% endraw %}{% echo "foo" %}{% enddisable %}'
end
end
end

View File

@@ -198,12 +198,6 @@ class StandardFiltersTest < Minitest::Test
assert_equal [{ "a" => 1 }, { "a" => 2 }, { "a" => 3 }, { "a" => 4 }], @filters.sort([{ "a" => 4 }, { "a" => 3 }, { "a" => 1 }, { "a" => 2 }], "a")
end
def test_sort_numeric
assert_equal ['1', '2', '3', '10'], @filters.sort_numeric(['10', '3', '2', '1'])
assert_equal [{ "a" => '1' }, { "a" => '2' }, { "a" => '3' }, { "a" => '10' }],
@filters.sort_numeric([{ "a" => '10' }, { "a" => '3' }, { "a" => '1' }, { "a" => '2' }], "a")
end
def test_sort_with_nils
assert_equal [1, 2, 3, 4, nil], @filters.sort([nil, 4, 3, 2, 1])
assert_equal [{ "a" => 1 }, { "a" => 2 }, { "a" => 3 }, { "a" => 4 }, {}], @filters.sort([{ "a" => 4 }, { "a" => 3 }, {}, { "a" => 1 }, { "a" => 2 }], "a")
@@ -298,10 +292,6 @@ class StandardFiltersTest < Minitest::Test
assert_equal [], @filters.sort_natural([], "a")
end
def test_sort_numeric_empty_array
assert_equal [], @filters.sort_numeric([], "a")
end
def test_sort_natural_invalid_property
foo = [
[1],

View File

@@ -89,12 +89,14 @@ class RenderTagTest < Minitest::Test
end
end
def test_sub_contexts_count_towards_the_same_recursion_limit
def test_includes_and_renders_count_towards_the_same_recursion_limit
Liquid::Template.file_system = StubFileSystem.new(
'loop_render' => '{% render "loop_render" %}',
'loop_render' => '{% render "loop_include" %}',
'loop_include' => '{% include "loop_render" %}'
)
assert_raises Liquid::StackLevelError do
Template.parse('{% render "loop_render" %}').render!
assert_raises Liquid::StackLevelError do
Template.parse('{% render "loop_include" %}').render!
end
end
@@ -146,23 +148,4 @@ class RenderTagTest < Minitest::Test
Liquid::Template.file_system = StubFileSystem.new('decr' => '{% decrement %}')
assert_template_result '-1-2-1', '{% decrement %}{% decrement %}{% render "decr" %}'
end
def test_includes_will_not_render_inside_render_tag
Liquid::Template.file_system = StubFileSystem.new(
'foo' => 'bar',
'test_include' => '{% include "foo" %}'
)
assert_template_result 'include usage is not allowed in this context', '{% render "test_include" %}'
end
def test_includes_will_not_render_inside_nested_sibling_tags
Liquid::Template.file_system = StubFileSystem.new(
'foo' => 'bar',
'nested_render_with_sibling_include' => '{% render "test_include" %}{% include "foo" %}',
'test_include' => '{% include "foo" %}'
)
assert_template_result 'include usage is not allowed in this contextinclude usage is not allowed in this context', '{% render "nested_render_with_sibling_include" %}'
end
end

View File

@@ -81,10 +81,7 @@ class TemplateTest < Minitest::Test
def test_lambda_is_called_once_from_persistent_assigns_over_multiple_parses_and_renders
t = Template.new
t.assigns['number'] = -> {
@global ||= 0
@global += 1
}
t.assigns['number'] = -> { @global ||= 0; @global += 1 }
assert_equal '1', t.parse("{{number}}").render!
assert_equal '1', t.parse("{{number}}").render!
assert_equal '1', t.render!
@@ -93,10 +90,7 @@ class TemplateTest < Minitest::Test
def test_lambda_is_called_once_from_custom_assigns_over_multiple_parses_and_renders
t = Template.new
assigns = { 'number' => -> {
@global ||= 0
@global += 1
} }
assigns = { 'number' => -> { @global ||= 0; @global += 1 } }
assert_equal '1', t.parse("{{number}}").render!(assigns)
assert_equal '1', t.parse("{{number}}").render!(assigns)
assert_equal '1', t.render!(assigns)
@@ -243,10 +237,7 @@ class TemplateTest < Minitest::Test
def test_exception_renderer_that_returns_string
exception = nil
handler = ->(e) {
exception = e
'<!-- error -->'
}
handler = ->(e) { exception = e; '<!-- error -->' }
output = Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_renderer: handler)
@@ -257,10 +248,7 @@ class TemplateTest < Minitest::Test
def test_exception_renderer_that_raises
exception = nil
assert_raises(Liquid::ZeroDivisionError) do
Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_renderer: ->(e) {
exception = e
raise
})
Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_renderer: ->(e) { exception = e; raise })
end
assert exception.is_a?(Liquid::ZeroDivisionError)
end

View File

@@ -9,7 +9,7 @@ require 'liquid.rb'
require 'liquid/profiler'
mode = :strict
if (env_mode = ENV['LIQUID_PARSER_MODE'])
if env_mode = ENV['LIQUID_PARSER_MODE']
puts "-- #{env_mode.upcase} ERROR MODE"
mode = env_mode.to_sym
end

View File

@@ -26,9 +26,9 @@ class ConditionUnitTest < Minitest::Test
assert_evaluates_true 1, '<=', 1
# negative numbers
assert_evaluates_true 1, '>', -1
assert_evaluates_true(-1, '<', 1)
assert_evaluates_true -1, '<', 1
assert_evaluates_true 1.0, '>', -1.0
assert_evaluates_true(-1.0, '<', 1.0)
assert_evaluates_true -1.0, '<', 1.0
end
def test_default_operators_evalute_false

View File

@@ -85,7 +85,7 @@ class ContextUnitTest < Minitest::Test
@context['date'] = Date.today
assert_equal Date.today, @context['date']
now = Time.now
now = DateTime.now
@context['datetime'] = now
assert_equal now, @context['datetime']
@@ -405,11 +405,7 @@ class ContextUnitTest < Minitest::Test
end
def test_lambda_is_called_once
@context['callcount'] = proc {
@global ||= 0
@global += 1
@global.to_s
}
@context['callcount'] = proc { @global ||= 0; @global += 1; @global.to_s }
assert_equal '1', @context['callcount']
assert_equal '1', @context['callcount']
@@ -419,11 +415,7 @@ class ContextUnitTest < Minitest::Test
end
def test_nested_lambda_is_called_once
@context['callcount'] = { "lambda" => proc {
@global ||= 0
@global += 1
@global.to_s
} }
@context['callcount'] = { "lambda" => proc { @global ||= 0; @global += 1; @global.to_s } }
assert_equal '1', @context['callcount.lambda']
assert_equal '1', @context['callcount.lambda']
@@ -433,11 +425,7 @@ class ContextUnitTest < Minitest::Test
end
def test_lambda_in_array_is_called_once
@context['callcount'] = [1, 2, proc {
@global ||= 0
@global += 1
@global.to_s
}, 4, 5]
@context['callcount'] = [1, 2, proc { @global ||= 0; @global += 1; @global.to_s }, 4, 5]
assert_equal '1', @context['callcount[2]']
assert_equal '1', @context['callcount[2]']

View File

@@ -1,36 +0,0 @@
# frozen_string_literal: true
require 'test_helper'
class DisabledTagsUnitTest < Minitest::Test
include Liquid
def test_disables_tag_specified
register = DisabledTags.new
register.disable(%w(foo bar)) do
assert_equal true, register.disabled?("foo")
assert_equal true, register.disabled?("bar")
assert_equal false, register.disabled?("unknown")
end
end
def test_disables_nested_tags
register = DisabledTags.new
register.disable(["foo"]) do
register.disable(["foo"]) do
assert_equal true, register.disabled?("foo")
assert_equal false, register.disabled?("bar")
end
register.disable(["bar"]) do
assert_equal true, register.disabled?("foo")
assert_equal true, register.disabled?("bar")
register.disable(["foo"]) do
assert_equal true, register.disabled?("foo")
assert_equal true, register.disabled?("bar")
end
end
assert_equal true, register.disabled?("foo")
assert_equal false, register.disabled?("bar")
end
end
end

View File

@@ -35,7 +35,7 @@ class TokenizerTest < Minitest::Test
def tokenize(source)
tokenizer = Liquid::Tokenizer.new(source)
tokens = []
while (t = tokenizer.shift)
while t = tokenizer.shift
tokens << t
end
tokens