Compare commits

..

3 Commits

Author SHA1 Message Date
Mike Angell
a2814443bb Prefer string interpolation in simple cases
Co-Authored-By: Dylan Thacker-Smith <dylan.smith@shopify.com>
2019-09-17 00:31:19 +10:00
Mike Angell
c193ce3dab Update rubocop config 2019-09-16 18:15:37 +10:00
Mike Angell
7abb8a3554 Enabled frozen string literals 2019-09-16 12:54:33 +10:00
23 changed files with 93 additions and 344 deletions

View File

@@ -6,6 +6,26 @@
# Note that changes in the inspected code, or installation of new
# versions of RuboCop, may require this file to be generated again.
# Offense count: 2
Lint/AmbiguousOperator:
Exclude:
- 'test/unit/condition_unit_test.rb'
# Offense count: 21
# Configuration parameters: AllowSafeAssignment.
Lint/AssignmentInCondition:
Exclude:
- 'lib/liquid/block_body.rb'
- 'lib/liquid/lexer.rb'
- 'lib/liquid/standardfilters.rb'
- 'lib/liquid/tags/for.rb'
- 'lib/liquid/tags/if.rb'
- 'lib/liquid/tags/raw.rb'
- 'lib/liquid/variable.rb'
- 'performance/profile.rb'
- 'test/test_helper.rb'
- 'test/unit/tokenizer_unit_test.rb'
# Offense count: 2
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
@@ -14,6 +34,17 @@ Lint/InheritException:
Exclude:
- 'lib/liquid/interrupts.rb'
# Offense count: 2
Lint/UselessAssignment:
Exclude:
- 'performance/shopify/database.rb'
# Offense count: 1
# Configuration parameters: CheckForMethodsWithNoSideEffects.
Lint/Void:
Exclude:
- 'lib/liquid/parse_context.rb'
# Offense count: 98
# Cop supports --auto-correct.
# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
@@ -45,4 +76,19 @@ Style/ClassVars:
Exclude:
- 'lib/liquid/condition.rb'
- 'lib/liquid/strainer.rb'
- 'lib/liquid/template.rb'
- 'lib/liquid/template.rb'
# Offense count: 1
# Configuration parameters: AllowCoercion.
Style/DateTime:
Exclude:
- 'test/unit/context_unit_test.rb'
# Offense count: 9
# Cop supports --auto-correct.
# Configuration parameters: AllowAsExpressionSeparator.
Style/Semicolon:
Exclude:
- 'test/integration/error_handling_test.rb'
- 'test/integration/template_test.rb'
- 'test/unit/context_unit_test.rb'

View File

@@ -78,7 +78,6 @@ require 'liquid/tokenizer'
require 'liquid/parse_context'
require 'liquid/partial_cache'
require 'liquid/usage'
require 'liquid/static_registers'
# Load all the tags of the standard library
#

View File

@@ -27,7 +27,7 @@ module Liquid
end
private def parse_for_liquid_tag(tokenizer, parse_context)
while (token = tokenizer.shift)
while token = tokenizer.shift
unless token.empty? || token =~ WhitespaceOrNothing
unless token =~ LiquidTagToken
# line isn't empty but didn't match tag syntax, yield and let the
@@ -36,7 +36,7 @@ module Liquid
end
tag_name = Regexp.last_match(1)
markup = Regexp.last_match(2)
unless (tag = registered_tags[tag_name])
unless tag = registered_tags[tag_name]
# end parsing if we reach an unknown tag and let the caller decide
# determine how to proceed
return yield tag_name, markup
@@ -52,7 +52,7 @@ module Liquid
end
private def parse_for_document(tokenizer, parse_context, &block)
while (token = tokenizer.shift)
while token = tokenizer.shift
next if token.empty?
case
when token.start_with?(TAGSTART)
@@ -74,7 +74,7 @@ module Liquid
next parse_for_liquid_tag(liquid_tag_tokenizer, parse_context, &block)
end
unless (tag = registered_tags[tag_name])
unless tag = registered_tags[tag_name]
# end parsing if we reach an unknown tag and let the caller decide
# determine how to proceed
return yield tag_name, markup
@@ -122,7 +122,7 @@ module Liquid
context.resource_limits.render_score += @nodelist.length
idx = 0
while (node = @nodelist[idx])
while node = @nodelist[idx]
previous_output_size = output.bytesize
case node

View File

@@ -18,17 +18,18 @@ module Liquid
attr_accessor :exception_renderer, :template_name, :partial, :global_filter, :strict_variables, :strict_filters
# rubocop:disable Metrics/ParameterLists
def self.build(environments: {}, outer_scope: {}, registers: {}, rethrow_errors: false, resource_limits: nil, static_environments: {})
new(environments, outer_scope, registers, rethrow_errors, resource_limits, static_environments)
def self.build(environments: {}, outer_scope: {}, registers: {}, rethrow_errors: false, resource_limits: nil, static_registers: {}, static_environments: {})
new(environments, outer_scope, registers, rethrow_errors, resource_limits, static_registers, static_environments)
end
def initialize(environments = {}, outer_scope = {}, registers = {}, rethrow_errors = false, resource_limits = nil, static_environments = {})
def initialize(environments = {}, outer_scope = {}, registers = {}, rethrow_errors = false, resource_limits = nil, static_registers = {}, static_environments = {})
@environments = [environments]
@environments.flatten!
@static_environments = [static_environments].flat_map(&:freeze).freeze
@scopes = [(outer_scope || {})]
@registers = registers
@static_registers = static_registers.freeze
@errors = []
@partial = false
@strict_variables = false
@@ -136,7 +137,7 @@ module Liquid
Context.build(
resource_limits: resource_limits,
static_environments: static_environments,
registers: StaticRegisters.new(registers)
static_registers: static_registers
).tap do |subcontext|
subcontext.base_scope_depth = base_scope_depth + 1
subcontext.exception_renderer = exception_renderer

View File

@@ -69,7 +69,7 @@ module Liquid
if include?(Enumerable)
blacklist += Enumerable.public_instance_methods
blacklist -= [:sort, :count, :first, :min, :max]
blacklist -= [:sort, :count, :first, :min, :max, :include?]
end
whitelist = [:to_liquid] + (public_instance_methods - blacklist)

View File

@@ -33,21 +33,15 @@ module Liquid
until @ss.eos?
@ss.skip(WHITESPACE_OR_NOTHING)
break if @ss.eos?
tok = if (t = @ss.scan(COMPARISON_OPERATOR))
[:comparison, t]
elsif (t = @ss.scan(SINGLE_STRING_LITERAL))
[:string, t]
elsif (t = @ss.scan(DOUBLE_STRING_LITERAL))
[:string, t]
elsif (t = @ss.scan(NUMBER_LITERAL))
[:number, t]
elsif (t = @ss.scan(IDENTIFIER))
[:id, t]
elsif (t = @ss.scan(DOTDOT))
[:dotdot, t]
tok = if t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t]
elsif t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t]
elsif t = @ss.scan(DOUBLE_STRING_LITERAL) then [:string, t]
elsif t = @ss.scan(NUMBER_LITERAL) then [:number, t]
elsif t = @ss.scan(IDENTIFIER) then [:id, t]
elsif t = @ss.scan(DOTDOT) then [:dotdot, t]
else
c = @ss.getch
if (s = SPECIALS[c])
if s = SPECIALS[c]
[s, c]
else
raise SyntaxError, "Unexpected character #{c}"

View File

@@ -21,6 +21,7 @@ module Liquid
@partial = value
@options = value ? partial_options : @template_options
@error_mode = @options[:error_mode] || Template.error_mode
value
end
def partial_options

View File

@@ -327,7 +327,7 @@ module Liquid
def date(input, format)
return input if format.to_s.empty?
return input unless (date = Utils.to_date(input))
return input unless date = Utils.to_date(input)
date.strftime(format.to_s)
end

View File

@@ -1,36 +0,0 @@
# frozen_string_literal: true
module Liquid
class StaticRegisters
attr_reader :static_registers, :registers
def initialize(registers = {})
@static_registers = registers.is_a?(StaticRegisters) ? registers.static_registers : registers.freeze
@registers = {}
end
def []=(key, value)
@registers[key] = value
end
def [](key)
if @registers.key?(key)
@registers[key]
else
@static_registers[key]
end
end
def delete(key)
@registers.delete(key)
end
def fetch(key, default = nil)
key?(key) ? self[key] : default
end
def key?(key)
@registers.key?(key) || @static_registers.key?(key)
end
end
end

View File

@@ -111,7 +111,7 @@ module Liquid
@reversed = p.id?('reversed')
while p.look(:id) && p.look(:colon, 1)
unless (attribute = p.id?('limit') || p.id?('offset'))
unless attribute = p.id?('limit') || p.id?('offset')
raise SyntaxError, options[:locale].t("errors.syntax.for_invalid_attribute")
end
p.consume

View File

@@ -94,7 +94,7 @@ module Liquid
def parse_binary_comparisons(p)
condition = parse_comparison(p)
first_condition = condition
while (op = (p.id?('and') || p.id?('or')))
while op = (p.id?('and') || p.id?('or'))
child_condition = parse_comparison(p)
condition.send(op, child_condition)
condition = child_condition
@@ -104,7 +104,7 @@ module Liquid
def parse_comparison(p)
a = Expression.parse(p.expression)
if (op = p.consume?(:comparison))
if op = p.consume?(:comparison)
b = Expression.parse(p.expression)
Condition.new(a, op, b)
else

View File

@@ -13,7 +13,7 @@ module Liquid
def parse(tokens)
@body = +''
while (token = tokens.shift)
while token = tokens.shift
if token =~ FullTokenPossiblyInvalid
@body << Regexp.last_match(1) if Regexp.last_match(1) != ""
return if block_delimiter == Regexp.last_match(2)

View File

@@ -110,7 +110,7 @@ module Liquid
filter_args = []
keyword_args = nil
unparsed_args.each do |a|
if (matches = a.match(JustTagAttributes))
if matches = a.match(JustTagAttributes)
keyword_args ||= {}
keyword_args[matches[1]] = Expression.parse(matches[2])
else

View File

@@ -15,7 +15,7 @@ profiler.run
end
end
if profile_type == :cpu && (graph_filename = ENV['GRAPH_FILENAME'])
if profile_type == :cpu && graph_filename = ENV['GRAPH_FILENAME']
File.open(graph_filename, 'w') do |f|
StackProf::Report.new(results).print_graphviz(nil, f)
end

View File

@@ -32,8 +32,8 @@ module Database
db['article'] = db['blog']['articles'].first
db['cart'] = {
'total_price' => db['line_items'].values.inject(0) { |sum, item| sum + item['line_price'] * item['quantity'] },
'item_count' => db['line_items'].values.inject(0) { |sum, item| sum + item['quantity'] },
'total_price' => db['line_items'].values.inject(0) { |sum, item| sum += item['line_price'] * item['quantity'] },
'item_count' => db['line_items'].values.inject(0) { |sum, item| sum += item['quantity'] },
'items' => db['line_items'].values,
}

View File

@@ -272,11 +272,4 @@ class DropsTest < Minitest::Test
assert_equal 'ProductDrop', Liquid::Template.parse("{{ product }}").render!('product' => ProductDrop.new)
assert_equal 'EnumerableDrop', Liquid::Template.parse('{{ collection }}').render!('collection' => EnumerableDrop.new)
end
def test_invokable_methods
assert_equal %w(to_liquid catchall user_input context texts).to_set, ProductDrop.invokable_methods
assert_equal %w(to_liquid scopes_as_array loop_pos scopes).to_set, ContextDrop.invokable_methods
assert_equal %w(to_liquid size max min first count).to_set, EnumerableDrop.invokable_methods
assert_equal %w(to_liquid max min sort count first).to_set, RealEnumerableDrop.invokable_methods
end
end # DropsTest

View File

@@ -211,10 +211,7 @@ class ErrorHandlingTest < Minitest::Test
def test_setting_default_exception_renderer
old_exception_renderer = Liquid::Template.default_exception_renderer
exceptions = []
Liquid::Template.default_exception_renderer = ->(e) {
exceptions << e
''
}
Liquid::Template.default_exception_renderer = ->(e) { exceptions << e; '' }
template = Liquid::Template.parse('This is a runtime error: {{ errors.argument_error }}')
output = template.render('errors' => ErrorDrop.new)
@@ -228,10 +225,7 @@ class ErrorHandlingTest < Minitest::Test
def test_exception_renderer_exposing_non_liquid_error
template = Liquid::Template.parse('This is a runtime error: {{ errors.runtime_error }}', line_numbers: true)
exceptions = []
handler = ->(e) {
exceptions << e
e.cause
}
handler = ->(e) { exceptions << e; e.cause }
output = template.render({ 'errors' => ErrorDrop.new }, exception_renderer: handler)

View File

@@ -81,10 +81,7 @@ class TemplateTest < Minitest::Test
def test_lambda_is_called_once_from_persistent_assigns_over_multiple_parses_and_renders
t = Template.new
t.assigns['number'] = -> {
@global ||= 0
@global += 1
}
t.assigns['number'] = -> { @global ||= 0; @global += 1 }
assert_equal '1', t.parse("{{number}}").render!
assert_equal '1', t.parse("{{number}}").render!
assert_equal '1', t.render!
@@ -93,10 +90,7 @@ class TemplateTest < Minitest::Test
def test_lambda_is_called_once_from_custom_assigns_over_multiple_parses_and_renders
t = Template.new
assigns = { 'number' => -> {
@global ||= 0
@global += 1
} }
assigns = { 'number' => -> { @global ||= 0; @global += 1 } }
assert_equal '1', t.parse("{{number}}").render!(assigns)
assert_equal '1', t.parse("{{number}}").render!(assigns)
assert_equal '1', t.render!(assigns)
@@ -243,10 +237,7 @@ class TemplateTest < Minitest::Test
def test_exception_renderer_that_returns_string
exception = nil
handler = ->(e) {
exception = e
'<!-- error -->'
}
handler = ->(e) { exception = e; '<!-- error -->' }
output = Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_renderer: handler)
@@ -257,10 +248,7 @@ class TemplateTest < Minitest::Test
def test_exception_renderer_that_raises
exception = nil
assert_raises(Liquid::ZeroDivisionError) do
Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_renderer: ->(e) {
exception = e
raise
})
Template.parse("{{ 1 | divided_by: 0 }}").render({}, exception_renderer: ->(e) { exception = e; raise })
end
assert exception.is_a?(Liquid::ZeroDivisionError)
end

View File

@@ -9,7 +9,7 @@ require 'liquid.rb'
require 'liquid/profiler'
mode = :strict
if (env_mode = ENV['LIQUID_PARSER_MODE'])
if env_mode = ENV['LIQUID_PARSER_MODE']
puts "-- #{env_mode.upcase} ERROR MODE"
mode = env_mode.to_sym
end

View File

@@ -26,9 +26,9 @@ class ConditionUnitTest < Minitest::Test
assert_evaluates_true 1, '<=', 1
# negative numbers
assert_evaluates_true 1, '>', -1
assert_evaluates_true(-1, '<', 1)
assert_evaluates_true -1, '<', 1
assert_evaluates_true 1.0, '>', -1.0
assert_evaluates_true(-1.0, '<', 1.0)
assert_evaluates_true -1.0, '<', 1.0
end
def test_default_operators_evalute_false

View File

@@ -85,7 +85,7 @@ class ContextUnitTest < Minitest::Test
@context['date'] = Date.today
assert_equal Date.today, @context['date']
now = Time.now
now = DateTime.now
@context['datetime'] = now
assert_equal now, @context['datetime']
@@ -405,11 +405,7 @@ class ContextUnitTest < Minitest::Test
end
def test_lambda_is_called_once
@context['callcount'] = proc {
@global ||= 0
@global += 1
@global.to_s
}
@context['callcount'] = proc { @global ||= 0; @global += 1; @global.to_s }
assert_equal '1', @context['callcount']
assert_equal '1', @context['callcount']
@@ -419,11 +415,7 @@ class ContextUnitTest < Minitest::Test
end
def test_nested_lambda_is_called_once
@context['callcount'] = { "lambda" => proc {
@global ||= 0
@global += 1
@global.to_s
} }
@context['callcount'] = { "lambda" => proc { @global ||= 0; @global += 1; @global.to_s } }
assert_equal '1', @context['callcount.lambda']
assert_equal '1', @context['callcount.lambda']
@@ -433,11 +425,7 @@ class ContextUnitTest < Minitest::Test
end
def test_lambda_in_array_is_called_once
@context['callcount'] = [1, 2, proc {
@global ||= 0
@global += 1
@global.to_s
}, 4, 5]
@context['callcount'] = [1, 2, proc { @global ||= 0; @global += 1; @global.to_s }, 4, 5]
assert_equal '1', @context['callcount[2]']
assert_equal '1', @context['callcount[2]']
@@ -530,23 +518,15 @@ class ContextUnitTest < Minitest::Test
registers = {
my_register: :my_value,
}
super_context = Context.new({}, {}, StaticRegisters.new(registers))
super_context.registers[:my_register] = :my_alt_value
super_context = Context.new({}, {}, registers)
subcontext = super_context.new_isolated_subcontext
assert_equal :my_value, subcontext.registers[:my_register]
assert_nil subcontext.registers[:my_register]
end
def test_new_isolated_subcontext_inherits_static_registers
super_context = Context.build(registers: { my_register: :my_value })
super_context = Context.build(static_registers: { my_register: :my_value })
subcontext = super_context.new_isolated_subcontext
assert_equal :my_value, subcontext.registers[:my_register]
end
def test_new_isolated_subcontext_registers_do_not_pollute_context
super_context = Context.build(registers: { my_register: :my_value })
subcontext = super_context.new_isolated_subcontext
subcontext.registers[:my_register] = :my_alt_value
assert_equal :my_value, super_context.registers[:my_register]
assert_equal :my_value, subcontext.static_registers[:my_register]
end
def test_new_isolated_subcontext_inherits_filters

View File

@@ -1,211 +0,0 @@
# frozen_string_literal: true
require 'test_helper'
class StaticRegistersUnitTest < Minitest::Test
include Liquid
def set
static_register = StaticRegisters.new
static_register[nil] = true
static_register[1] = :one
static_register[:one] = "one"
static_register["two"] = "three"
static_register["two"] = 3
static_register[false] = nil
assert_equal({ nil => true, 1 => :one, :one => "one", "two" => 3, false => nil }, static_register.registers)
static_register
end
def test_get
static_register = set
assert_equal true, static_register[nil]
assert_equal :one, static_register[1]
assert_equal "one", static_register[:one]
assert_equal 3, static_register["two"]
assert_nil static_register[false]
assert_nil static_register["unknown"]
end
def test_delete
static_register = set
assert_equal true, static_register.delete(nil)
assert_equal :one, static_register.delete(1)
assert_equal "one", static_register.delete(:one)
assert_equal 3, static_register.delete("two")
assert_nil static_register.delete(false)
assert_nil static_register.delete("unknown")
assert_equal({}, static_register.registers)
end
def test_fetch
static_register = set
assert_equal true, static_register.fetch(nil)
assert_equal :one, static_register.fetch(1)
assert_equal "one", static_register.fetch(:one)
assert_equal 3, static_register.fetch("two")
assert_nil static_register.fetch(false)
assert_nil static_register.fetch("unknown")
end
def test_fetch_default
static_register = StaticRegisters.new
assert_equal true, static_register.fetch(nil, true)
assert_equal :one, static_register.fetch(1, :one)
assert_equal "one", static_register.fetch(:one, "one")
assert_equal 3, static_register.fetch("two", 3)
assert_nil static_register.fetch(false, nil)
end
def test_key
static_register = set
assert_equal true, static_register.key?(nil)
assert_equal true, static_register.key?(1)
assert_equal true, static_register.key?(:one)
assert_equal true, static_register.key?("two")
assert_equal true, static_register.key?(false)
assert_equal false, static_register.key?("unknown")
assert_equal false, static_register.key?(true)
end
def set_with_static
static_register = StaticRegisters.new(nil => true, 1 => :one, :one => "one", "two" => 3, false => nil)
static_register[nil] = false
static_register["two"] = 4
static_register[true] = "foo"
assert_equal({ nil => true, 1 => :one, :one => "one", "two" => 3, false => nil }, static_register.static_registers)
assert_equal({ nil => false, "two" => 4, true => "foo" }, static_register.registers)
static_register
end
def test_get_with_static
static_register = set_with_static
assert_equal false, static_register[nil]
assert_equal :one, static_register[1]
assert_equal "one", static_register[:one]
assert_equal 4, static_register["two"]
assert_equal "foo", static_register[true]
assert_nil static_register[false]
end
def test_delete_with_static
static_register = set_with_static
assert_equal false, static_register.delete(nil)
assert_equal 4, static_register.delete("two")
assert_equal "foo", static_register.delete(true)
assert_nil static_register.delete("unknown")
assert_nil static_register.delete(:one)
assert_equal({}, static_register.registers)
assert_equal({ nil => true, 1 => :one, :one => "one", "two" => 3, false => nil }, static_register.static_registers)
end
def test_fetch_with_static
static_register = set_with_static
assert_equal false, static_register.fetch(nil)
assert_equal :one, static_register.fetch(1)
assert_equal "one", static_register.fetch(:one)
assert_equal 4, static_register.fetch("two")
assert_equal "foo", static_register.fetch(true)
assert_nil static_register.fetch(false)
end
def test_key_with_static
static_register = set_with_static
assert_equal true, static_register.key?(nil)
assert_equal true, static_register.key?(1)
assert_equal true, static_register.key?(:one)
assert_equal true, static_register.key?("two")
assert_equal true, static_register.key?(false)
assert_equal false, static_register.key?("unknown")
assert_equal true, static_register.key?(true)
end
def test_static_register_frozen
static_register = set_with_static
static = static_register.static_registers
assert_raises(RuntimeError) do
static["two"] = "foo"
end
assert_raises(RuntimeError) do
static["unknown"] = "foo"
end
assert_raises(RuntimeError) do
static.delete("two")
end
end
def test_new_static_retains_static
static_register = StaticRegisters.new(nil => true, 1 => :one, :one => "one", "two" => 3, false => nil)
static_register["one"] = 1
static_register["two"] = 2
static_register["three"] = 3
new_register = StaticRegisters.new(static_register)
assert_equal({}, new_register.registers)
new_register["one"] = 4
new_register["two"] = 5
new_register["three"] = 6
newest_register = StaticRegisters.new(new_register)
assert_equal({}, newest_register.registers)
newest_register["one"] = 7
newest_register["two"] = 8
newest_register["three"] = 9
assert_equal({ "one" => 1, "two" => 2, "three" => 3 }, static_register.registers)
assert_equal({ "one" => 4, "two" => 5, "three" => 6 }, new_register.registers)
assert_equal({ "one" => 7, "two" => 8, "three" => 9 }, newest_register.registers)
assert_equal({ nil => true, 1 => :one, :one => "one", "two" => 3, false => nil }, static_register.static_registers)
assert_equal({ nil => true, 1 => :one, :one => "one", "two" => 3, false => nil }, new_register.static_registers)
assert_equal({ nil => true, 1 => :one, :one => "one", "two" => 3, false => nil }, newest_register.static_registers)
end
def test_multiple_instances_are_unique
static_register = StaticRegisters.new(nil => true, 1 => :one, :one => "one", "two" => 3, false => nil)
static_register["one"] = 1
static_register["two"] = 2
static_register["three"] = 3
new_register = StaticRegisters.new(foo: :bar)
assert_equal({}, new_register.registers)
new_register["one"] = 4
new_register["two"] = 5
new_register["three"] = 6
newest_register = StaticRegisters.new(bar: :foo)
assert_equal({}, newest_register.registers)
newest_register["one"] = 7
newest_register["two"] = 8
newest_register["three"] = 9
assert_equal({ "one" => 1, "two" => 2, "three" => 3 }, static_register.registers)
assert_equal({ "one" => 4, "two" => 5, "three" => 6 }, new_register.registers)
assert_equal({ "one" => 7, "two" => 8, "three" => 9 }, newest_register.registers)
assert_equal({ nil => true, 1 => :one, :one => "one", "two" => 3, false => nil }, static_register.static_registers)
assert_equal({ foo: :bar }, new_register.static_registers)
assert_equal({ bar: :foo }, newest_register.static_registers)
end
end

View File

@@ -35,7 +35,7 @@ class TokenizerTest < Minitest::Test
def tokenize(source)
tokenizer = Liquid::Tokenizer.new(source)
tokens = []
while (t = tokenizer.shift)
while t = tokenizer.shift
tokens << t
end
tokens